{"level":"info","ts":1789173985.920564,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8314452ad76b7df3b0fc2efa94d5e27f","span_id":"067323fdc9b3e5ca","x-request-id":"a623f44f-969c-43ad-ad6a-9d6191f29408"} {"level":"info","ts":1789173985.9249651,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"c507efa8-2d5a-4019-a1d5-67a9239d24b2","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9282997,"caller":"legacymigration/reconciler.go:147","msg":"creating ExternalProvider","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"c507efa8-2d5a-4019-a1d5-67a9239d24b2","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9329724,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"ffb511c8-e514-4d66-9f95-c4693a224359","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9330578,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"ffb511c8-e514-4d66-9f95-c4693a224359","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789173985.9330099,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d"} {"level":"info","ts":1789173985.9418674,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"5072d03b-9073-456c-aef6-6bd0a5c75ab2"} {"level":"error","ts":1789173985.942112,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"c507efa8-2d5a-4019-a1d5-67a9239d24b2","error":"failed to apply ExternalProvider: externalproviders.inference.opendatahub.io \"e2e-legacy-migration-964f7006\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173985.9421942,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"6f32e40f-7096-438b-9e7f-d5ed7d10a37f","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9422598,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"6f32e40f-7096-438b-9e7f-d5ed7d10a37f","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789173985.942137,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"6bca5745-0620-4661-a0a2-ea7f3e05f722","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9422884,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"6bca5745-0620-4661-a0a2-ea7f3e05f722","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"info","ts":1789173985.9423218,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d307d12e-e7f6-4664-9131-b924a9162ae8","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.942373,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d307d12e-e7f6-4664-9131-b924a9162ae8","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"error","ts":1789173985.9425101,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-964f7006\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:84\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173985.9425647,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9472876,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"2a6ce286-181e-41a6-9e34-7ae3ae8fc4f4","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9480336,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"5072d03b-9073-456c-aef6-6bd0a5c75ab2","provider":"e2e-legacy-migration-964f7006","phase":"Pending"} {"level":"info","ts":1789173985.9481277,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"b9e7caaf-c185-44f8-b3be-c34c5f73f522","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.948187,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"b9e7caaf-c185-44f8-b3be-c34c5f73f522","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"info","ts":1789173985.9616334,"caller":"externalprovider/reconciler.go:210","msg":"Creating resource","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d","kind":"ServiceEntry","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9619162,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789173985.9620788,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"info","ts":1789173985.962121,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"info","ts":1789173985.9621425,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"2a6ce286-181e-41a6-9e34-7ae3ae8fc4f4","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9621625,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","EoS":true} {"level":"info","ts":1789173985.9622347,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","header":"x-maas-subscription"} {"level":"info","ts":1789173985.962266,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","header":"x-maas-username"} {"level":"info","ts":1789173985.9622915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","header":"x-maas-group"} {"level":"info","ts":1789173985.9623268,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789173985.9623823,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"error","ts":1789173985.9627028,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"5072d03b-9073-456c-aef6-6bd0a5c75ab2","error":"ExternalModel \"e2e-legacy-migration-964f7006\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-964f7006\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173985.9628599,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"29403091-04fa-4f1e-bc5e-fa82f2c80743"} {"level":"info","ts":1789173985.9629226,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"29403091-04fa-4f1e-bc5e-fa82f2c80743","provider":"e2e-legacy-migration-964f7006","phase":"Pending"} {"level":"info","ts":1789173985.96294,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"95ac1cd9-a3ea-4d10-8d2c-ca3aa6c8a1ab","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9632869,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"95ac1cd9-a3ea-4d10-8d2c-ca3aa6c8a1ab","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"info","ts":1789173985.9635234,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"info","ts":1789173985.9635808,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"info","ts":1789173985.963624,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","EoS":false} {"level":"info","ts":1789173985.963666,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"info","ts":1789173985.963713,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4","EoS":true} {"level":"info","ts":1789173985.9637432,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2b69ecdb08830034cd15a7015d9c24dd","span_id":"c25e46330b147de5","x-request-id":"3c96d2c2-41e4-4405-a9c9-a651161566c4"} {"level":"error","ts":1789173985.9720633,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"2a6ce286-181e-41a6-9e34-7ae3ae8fc4f4","error":"failed to apply ExternalModel: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-964f7006\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789173985.9723327,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"29403091-04fa-4f1e-bc5e-fa82f2c80743","error":"ExternalModel \"e2e-legacy-migration-964f7006\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-964f7006\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173985.9724362,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d0d6d331-44e4-4f84-a5a7-f88b09fcc7d9"} {"level":"info","ts":1789173985.9724917,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d0d6d331-44e4-4f84-a5a7-f88b09fcc7d9","provider":"e2e-legacy-migration-964f7006","phase":"Pending"} {"level":"error","ts":1789173985.9812834,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d0d6d331-44e4-4f84-a5a7-f88b09fcc7d9","error":"ExternalModel \"e2e-legacy-migration-964f7006\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-964f7006\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173985.9828498,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"2fc4382e-4f08-4925-9f1f-f8084ba366b2","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9865384,"caller":"externalprovider/reconciler.go:210","msg":"Creating resource","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d","kind":"DestinationRule","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9870515,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"2fc4382e-4f08-4925-9f1f-f8084ba366b2","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9952152,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d312202d-808a-4b76-a930-91f779aa35b7","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9952981,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"d312202d-808a-4b76-a930-91f779aa35b7","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"info","ts":1789173985.9954145,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"040d0775-7578-4a57-831e-58099dab4e8b"} {"level":"info","ts":1789173985.9954884,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"040d0775-7578-4a57-831e-58099dab4e8b","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173985.9957938,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"bd9fb9af-96c8-4afa-be7d-51defcb16c60","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173985.9958606,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"bd9fb9af-96c8-4afa-be7d-51defcb16c60","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789173986.0002687,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"2fc4382e-4f08-4925-9f1f-f8084ba366b2","provider":"openai","model":"e2e-legacy-migration-964f7006","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789173986.0066373,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789173986.0068126,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"info","ts":1789173986.0068674,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"info","ts":1789173986.0069613,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","EoS":true} {"level":"info","ts":1789173986.00701,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","header":"x-maas-username"} {"level":"info","ts":1789173986.0070426,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","header":"x-maas-subscription"} {"level":"info","ts":1789173986.0070775,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","header":"x-maas-group"} {"level":"info","ts":1789173986.0071146,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789173986.0071628,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"info","ts":1789173986.010881,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"info","ts":1789173986.0109432,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"info","ts":1789173986.0109866,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","EoS":false} {"level":"info","ts":1789173986.0110214,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"info","ts":1789173986.0110693,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d","EoS":true} {"level":"info","ts":1789173986.0110986,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6ba70480333134d9cb948915440a965a","span_id":"c00efb1e63571717","x-request-id":"1793ad1a-b4cc-44b9-ab1d-75e9fc17ea6d"} {"level":"error","ts":1789173986.0113473,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-964f7006\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789173986.0114317,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fb13c15b-4602-4634-8a62-8f487e895a4d","error":"failed to apply DestinationRule: destinationrules.networking.istio.io \"e2e-legacy-migration-964f7006\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173986.0115223,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"bff11507-692c-447a-b931-0cf7eb47ddcd"} {"level":"info","ts":1789173986.011603,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"bff11507-692c-447a-b931-0cf7eb47ddcd","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173986.0142891,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"7bbe1ba0-d93d-438a-be91-9cf65f31f6eb","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173986.0143793,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"7bbe1ba0-d93d-438a-be91-9cf65f31f6eb","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"error","ts":1789173986.027806,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"040d0775-7578-4a57-831e-58099dab4e8b","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-964f7006\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789173986.0279024,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"040d0775-7578-4a57-831e-58099dab4e8b","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-964f7006\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173986.0279958,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65034332-3ecc-4d9c-93b8-40f73421e2e6"} {"level":"info","ts":1789173986.028086,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65034332-3ecc-4d9c-93b8-40f73421e2e6","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173986.0299253,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"bff11507-692c-447a-b931-0cf7eb47ddcd","service":"e2e-legacy-migration-964f7006","serviceEntry":"e2e-legacy-migration-964f7006","destinationRule":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173986.040102,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fa6fbeb0-74df-41a6-8b00-ab87a95088b3","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173986.0402136,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"fa6fbeb0-74df-41a6-8b00-ab87a95088b3","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"info","ts":1789173986.04054,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65034332-3ecc-4d9c-93b8-40f73421e2e6","httpRoute":"e2e-legacy-migration-964f7006","providers":["e2e-legacy-migration-964f7006"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789173986.045511,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65034332-3ecc-4d9c-93b8-40f73421e2e6","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-964f7006\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173986.0456452,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"84596ef8-d568-4702-b190-b6dcb386c945"} {"level":"info","ts":1789173986.045726,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"84596ef8-d568-4702-b190-b6dcb386c945","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173986.0549366,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"84596ef8-d568-4702-b190-b6dcb386c945","httpRoute":"e2e-legacy-migration-964f7006","providers":["e2e-legacy-migration-964f7006"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789173986.0624053,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"6ddc6581-a5df-4780-8427-895c72e01e9d","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173986.062488,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"6ddc6581-a5df-4780-8427-895c72e01e9d","modelName":"e2e-legacy-migration-964f7006","resolvedRefs":1} {"level":"error","ts":1789173986.0670083,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"84596ef8-d568-4702-b190-b6dcb386c945","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-964f7006\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789173986.067154,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65be47ad-cd7f-4ff7-8017-e6128fb45b72"} {"level":"info","ts":1789173986.0672479,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65be47ad-cd7f-4ff7-8017-e6128fb45b72","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173986.0782983,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"65be47ad-cd7f-4ff7-8017-e6128fb45b72","httpRoute":"e2e-legacy-migration-964f7006","providers":["e2e-legacy-migration-964f7006"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789173986.2005117,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"7affe1cb-43d0-4c1f-8927-94ce6255842e","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173986.2073078,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"7affe1cb-43d0-4c1f-8927-94ce6255842e","name":"e2e-legacy-migration-964f7006"} {"level":"info","ts":1789173986.221142,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"7affe1cb-43d0-4c1f-8927-94ce6255842e","provider":"openai","model":"e2e-legacy-migration-964f7006","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789173988.865331,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"44a88664-504c-4c26-97c4-96e9981eb25a","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173988.8781083,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"05d45d19-14e2-4e32-a892-90df61c321b9","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173988.8781657,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"05d45d19-14e2-4e32-a892-90df61c321b9","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173988.8782053,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"036b0709-8994-45e9-950d-dc464a09f0d1"} {"level":"info","ts":1789173988.8810296,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"8834ca84-28e8-4688-ba42-f2339ff7c446","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173988.8810732,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"8834ca84-28e8-4688-ba42-f2339ff7c446","name":"e2e-legacy-migration-964f7006","namespace":"llm"} {"level":"info","ts":1789173988.8811135,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-964f7006","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-964f7006","reconcileID":"7ede0f93-77b0-4899-8f35-1eca970bcdf4"} {"level":"info","ts":1789173989.298164,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789173989.2982848,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.2983012,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.298318,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","EoS":true} {"level":"info","ts":1789173989.2983503,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","header":"x-maas-group"} {"level":"info","ts":1789173989.2983634,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","header":"x-maas-username"} {"level":"info","ts":1789173989.2983716,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","header":"x-maas-subscription"} {"level":"info","ts":1789173989.2983913,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789173989.2984037,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","modelName":"e2e-external-model"} {"level":"info","ts":1789173989.298421,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789173989.2984502,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789173989.2984717,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","authType":"apikey"} {"level":"info","ts":1789173989.298505,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.3492203,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.3492467,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.3492634,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","EoS":false} {"level":"info","ts":1789173989.349277,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.3493066,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca","EoS":true} {"level":"info","ts":1789173989.3493133,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b97b7c2473eb930296a7ba0b3c8490f1","span_id":"d2dec9472ac6dc84","x-request-id":"d4a7fe14-87f1-495e-a304-fd129808bcca"} {"level":"info","ts":1789173989.3971615,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789173989.3973632,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.3974135,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.3974357,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","EoS":true} {"level":"info","ts":1789173989.3974738,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","header":"x-maas-group"} {"level":"info","ts":1789173989.3975608,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","header":"x-maas-subscription"} {"level":"info","ts":1789173989.3975756,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","header":"x-maas-username"} {"level":"info","ts":1789173989.3975966,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789173989.3976233,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.4411817,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.4412131,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.441234,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","EoS":false} {"level":"info","ts":1789173989.4412506,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.4412892,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63","EoS":true} {"level":"info","ts":1789173989.4413037,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bcb76ff2f5e5540c4dd0ab52e9bb7811","span_id":"97a13d8a6af3bd62","x-request-id":"b197a8a2-1db0-4a8d-861d-284bed19ad63"} {"level":"info","ts":1789173989.4866502,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789173989.4867918,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173989.486857,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173989.4868762,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18","EoS":true} {"level":"info","ts":1789173989.4869063,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18","header":"x-maas-subscription"} {"level":"info","ts":1789173989.4869194,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18","header":"x-maas-group"} {"level":"info","ts":1789173989.486928,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18","header":"x-maas-username"} {"level":"info","ts":1789173989.4869633,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173989.5205855,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173989.5206115,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173989.520626,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18","EoS":false} {"level":"info","ts":1789173989.5206397,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173989.520672,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18","EoS":true} {"level":"info","ts":1789173989.5206847,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"86eb1f813069b0cb48b14b45d84cbd25","span_id":"87e558b2e86e7a5a","x-request-id":"d09ab399-43c4-4f91-bf79-f981210d4d18"} {"level":"info","ts":1789173990.3385715,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"267cd191-8dff-44a4-ac2f-333c6c7e4db3"} {"level":"info","ts":1789173990.3385758,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"81fb94da-c808-4c05-82b1-48f81a2d9955","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789173990.3386059,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"81fb94da-c808-4c05-82b1-48f81a2d9955","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789173990.480885,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"6056458c-ecda-4e47-ba3f-75b38ca41613","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789173990.4809194,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"6056458c-ecda-4e47-ba3f-75b38ca41613","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789173990.480941,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"35a31fd5-9025-4224-802b-fa5c0282d813"} {"level":"info","ts":1789173990.5956483,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7aaae5a0-729b-4c23-a1e3-76d839cb9253","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789173990.5957832,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7aaae5a0-729b-4c23-a1e3-76d839cb9253","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789173991.4041283,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789174003.9331872,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789174003.9333737,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7e2d1ba4fef48aa11068ba850565a1d8","span_id":"d23b42c272dcebe1","x-request-id":"10488d0a-8d69-4120-9368-39d7f3a2c179"} {"level":"info","ts":1789174003.933391,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7e2d1ba4fef48aa11068ba850565a1d8","span_id":"d23b42c272dcebe1","x-request-id":"10488d0a-8d69-4120-9368-39d7f3a2c179"} {"level":"info","ts":1789174003.933407,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7e2d1ba4fef48aa11068ba850565a1d8","span_id":"d23b42c272dcebe1","x-request-id":"10488d0a-8d69-4120-9368-39d7f3a2c179","EoS":true} {"level":"info","ts":1789174003.9334693,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7e2d1ba4fef48aa11068ba850565a1d8","span_id":"d23b42c272dcebe1","x-request-id":"10488d0a-8d69-4120-9368-39d7f3a2c179"} {"level":"info","ts":1789174004.093612,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789174004.093707,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"859a794731c351d515b9d03df7b948d4","span_id":"fe16d66260d8d892","x-request-id":"93e7a7c5-c249-4075-8cad-04a0833f96ae"} {"level":"info","ts":1789174004.0937195,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"859a794731c351d515b9d03df7b948d4","span_id":"fe16d66260d8d892","x-request-id":"93e7a7c5-c249-4075-8cad-04a0833f96ae"} {"level":"info","ts":1789174004.093732,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"859a794731c351d515b9d03df7b948d4","span_id":"fe16d66260d8d892","x-request-id":"93e7a7c5-c249-4075-8cad-04a0833f96ae","EoS":true} {"level":"info","ts":1789174004.0937996,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"859a794731c351d515b9d03df7b948d4","span_id":"fe16d66260d8d892","x-request-id":"93e7a7c5-c249-4075-8cad-04a0833f96ae"} {"level":"info","ts":1789174004.2490191,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789174004.2491152,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6357a1c05d35c44cee0f7191e8734a40","span_id":"0c85cfcdb2280bb8","x-request-id":"0debcf80-2e07-44d0-a541-a9146342d38f"} {"level":"info","ts":1789174004.2491264,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6357a1c05d35c44cee0f7191e8734a40","span_id":"0c85cfcdb2280bb8","x-request-id":"0debcf80-2e07-44d0-a541-a9146342d38f"} {"level":"info","ts":1789174004.2491379,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6357a1c05d35c44cee0f7191e8734a40","span_id":"0c85cfcdb2280bb8","x-request-id":"0debcf80-2e07-44d0-a541-a9146342d38f","EoS":true} {"level":"info","ts":1789174004.2491853,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6357a1c05d35c44cee0f7191e8734a40","span_id":"0c85cfcdb2280bb8","x-request-id":"0debcf80-2e07-44d0-a541-a9146342d38f"} {"level":"error","ts":1789174006.4050906,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789174021.4064364,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175061.1379528,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175061.1380622,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.1380737,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.1380851,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","EoS":true} {"level":"info","ts":1789175061.1381123,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","header":"x-maas-group"} {"level":"info","ts":1789175061.1381226,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","header":"x-maas-username"} {"level":"info","ts":1789175061.1381276,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","header":"x-maas-subscription"} {"level":"info","ts":1789175061.1381428,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789175061.1381686,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.1391532,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.139175,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.1391883,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","EoS":false} {"level":"info","ts":1789175061.1391988,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.139225,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4","EoS":true} {"level":"info","ts":1789175061.1392393,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"83bfa1b512bf2e04e854175be60c0bf3","span_id":"2e9667942e5d05ab","x-request-id":"3fdf0ea1-33af-4f72-ac5b-e046680231b4"} {"level":"info","ts":1789175061.177023,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175061.1771057,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"info","ts":1789175061.1771185,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"info","ts":1789175061.1771295,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","EoS":true} {"level":"info","ts":1789175061.177152,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","header":"x-maas-subscription"} {"level":"info","ts":1789175061.1771617,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","header":"x-maas-username"} {"level":"info","ts":1789175061.1771674,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","header":"x-maas-group"} {"level":"info","ts":1789175061.1771817,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789175061.177206,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"info","ts":1789175061.178057,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"info","ts":1789175061.1780746,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"info","ts":1789175061.1780906,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","EoS":false} {"level":"info","ts":1789175061.1781023,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"info","ts":1789175061.1781297,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0","EoS":true} {"level":"info","ts":1789175061.1781437,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a86602f54c9c7d1695e8023c13b3a52b","span_id":"a1e466994771590a","x-request-id":"83d7f7ed-ec13-4bfb-9715-5a60694a2cf0"} {"level":"error","ts":1789175071.5337944,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175389.2859027,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175389.2860186,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.2860305,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.2860432,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","EoS":true} {"level":"info","ts":1789175389.286071,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","header":"x-maas-group"} {"level":"info","ts":1789175389.2860818,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","header":"x-maas-subscription"} {"level":"info","ts":1789175389.2860878,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","header":"x-maas-username"} {"level":"info","ts":1789175389.286105,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175389.2861362,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.3002613,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.3002923,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.3003113,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","EoS":false} {"level":"info","ts":1789175389.300326,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.3003612,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c","EoS":true} {"level":"info","ts":1789175389.3003693,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0102aa04e48f04756146f4a5341804c0","span_id":"9c37426932ef8e69","x-request-id":"fcfcda17-9633-4480-9829-ebbd95dbf96c"} {"level":"info","ts":1789175389.444226,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175389.4443774,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.4443984,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.4444172,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","EoS":true} {"level":"info","ts":1789175389.4444737,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","header":"x-maas-username"} {"level":"info","ts":1789175389.44449,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","header":"x-maas-group"} {"level":"info","ts":1789175389.4444978,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","header":"x-maas-subscription"} {"level":"info","ts":1789175389.4445188,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175389.444558,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.4469442,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.4469764,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.4469922,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","EoS":false} {"level":"info","ts":1789175389.4470053,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.4470396,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a","EoS":true} {"level":"info","ts":1789175389.4470546,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"980382c092ae0e195198b543b438f41c","span_id":"9d81effdcc177d94","x-request-id":"29b51818-1ac2-4dfc-8208-8a166395a32a"} {"level":"info","ts":1789175389.587849,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175389.5879583,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.5879722,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.5880256,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","EoS":true} {"level":"info","ts":1789175389.5880647,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","header":"x-maas-subscription"} {"level":"info","ts":1789175389.5880766,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","header":"x-maas-group"} {"level":"info","ts":1789175389.5880818,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","header":"x-maas-username"} {"level":"info","ts":1789175389.5880988,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175389.5881324,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.589149,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.5891697,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.5891826,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","EoS":false} {"level":"info","ts":1789175389.589194,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.5892174,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9","EoS":true} {"level":"info","ts":1789175389.5892265,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b90dba0a27b95a887176d7b3bc2de9f9","span_id":"aa97d884ce410b5d","x-request-id":"e2c0df03-d0b5-42b0-8490-c3f60da5c0e9"} {"level":"info","ts":1789175389.7242997,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175389.7244122,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.7244277,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.7244403,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","EoS":true} {"level":"info","ts":1789175389.724514,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","header":"x-maas-group"} {"level":"info","ts":1789175389.7245314,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","header":"x-maas-subscription"} {"level":"info","ts":1789175389.724539,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","header":"x-maas-username"} {"level":"info","ts":1789175389.7245555,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175389.7245843,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.725587,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.7256074,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.725621,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","EoS":false} {"level":"info","ts":1789175389.7256386,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.7256625,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f","EoS":true} {"level":"info","ts":1789175389.725672,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"aaacf55d0cdd7e04c519af60dbe099db","span_id":"c537853150f17dd4","x-request-id":"0df816ba-32d2-46e0-aa54-7a2ac7cb971f"} {"level":"info","ts":1789175389.8650484,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175389.8651695,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"info","ts":1789175389.8651857,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"info","ts":1789175389.8652048,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","EoS":true} {"level":"info","ts":1789175389.8652442,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","header":"x-maas-username"} {"level":"info","ts":1789175389.865259,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","header":"x-maas-subscription"} {"level":"info","ts":1789175389.8652675,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","header":"x-maas-group"} {"level":"info","ts":1789175389.8652885,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175389.8653438,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"info","ts":1789175389.8663456,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"info","ts":1789175389.8663597,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"info","ts":1789175389.8663726,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","EoS":false} {"level":"info","ts":1789175389.8663886,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"info","ts":1789175389.8664136,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26","EoS":true} {"level":"info","ts":1789175389.8664224,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b28e2cebb0b044d75ea77cd1392d67ec","span_id":"3be87f012e55a9b0","x-request-id":"03480187-7ace-4715-aec3-7b8d06cdfe26"} {"level":"error","ts":1789175401.570244,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175403.8889554,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175403.8890705,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.8890839,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.889096,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","EoS":true} {"level":"info","ts":1789175403.8891237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","header":"x-maas-group"} {"level":"info","ts":1789175403.8891342,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","header":"x-maas-username"} {"level":"info","ts":1789175403.8891392,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","header":"x-maas-subscription"} {"level":"info","ts":1789175403.8891556,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175403.8891833,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.891468,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.8914971,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.891519,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","EoS":false} {"level":"info","ts":1789175403.891531,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.8915563,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822","EoS":true} {"level":"info","ts":1789175403.8915744,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1f741490ed2c205aeac77edf8bc530a3","span_id":"5b9a2416e51e8ecd","x-request-id":"a1af39fb-f310-4850-a563-8a3e0ec8a822"} {"level":"info","ts":1789175403.926586,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175403.9267027,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175403.9267159,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175403.926727,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","EoS":true} {"level":"info","ts":1789175403.9267528,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","header":"x-maas-username"} {"level":"info","ts":1789175403.926824,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","header":"x-maas-subscription"} {"level":"info","ts":1789175403.9268322,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","header":"x-maas-group"} {"level":"info","ts":1789175403.926852,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175403.9268816,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175403.927931,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175403.9279523,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175403.927965,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","EoS":false} {"level":"info","ts":1789175403.9279757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175403.928,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa","EoS":true} {"level":"info","ts":1789175403.9280095,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cc1e6588a93108c4745405b64557ec66","span_id":"c6f7341f8c9217fe","x-request-id":"6ac9854e-7479-4bd5-85c3-6bfc847866fa"} {"level":"info","ts":1789175404.0354028,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175404.0355158,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b"} {"level":"info","ts":1789175404.0374331,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b"} {"level":"info","ts":1789175404.0374534,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b"} {"level":"info","ts":1789175404.0374663,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b","EoS":false} {"level":"info","ts":1789175404.037482,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b"} {"level":"info","ts":1789175404.0375106,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b","EoS":true} {"level":"info","ts":1789175404.0375245,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"be427227d1ae93d2b5e2a87520a43cae","span_id":"8db50ecd2f4148eb","x-request-id":"24e1a506-c99d-4ddf-bbcb-4c4c9416cc9b"} {"level":"info","ts":1789175404.777741,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175404.7778945,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175404.7779088,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175404.7779212,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","EoS":true} {"level":"info","ts":1789175404.777947,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","header":"x-maas-group"} {"level":"info","ts":1789175404.7779589,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","header":"x-maas-subscription"} {"level":"info","ts":1789175404.7779682,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","header":"x-maas-username"} {"level":"info","ts":1789175404.7779882,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175404.7780201,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175404.7789161,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175404.778936,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175404.7789493,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","EoS":false} {"level":"info","ts":1789175404.7789593,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175404.778999,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e","EoS":true} {"level":"info","ts":1789175404.7790089,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9743f137ab6444d55b6e820ce8f75eb5","span_id":"5ec822cb319905f0","x-request-id":"f1ed20f7-c308-4742-baee-05bd45e9db5e"} {"level":"info","ts":1789175405.695444,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175405.695564,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175405.6955795,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175405.6955926,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","EoS":true} {"level":"info","ts":1789175405.6956196,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","header":"x-maas-group"} {"level":"info","ts":1789175405.6956294,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","header":"x-maas-username"} {"level":"info","ts":1789175405.6956348,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","header":"x-maas-subscription"} {"level":"info","ts":1789175405.6956518,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175405.6956758,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175405.6965687,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175405.6965911,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175405.6966105,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","EoS":false} {"level":"info","ts":1789175405.6966271,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175405.6966655,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41","EoS":true} {"level":"info","ts":1789175405.6966772,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f44731a20d4833fa55d8efa525e4f272","span_id":"d5b3f46fc50c8241","x-request-id":"09e61c66-ca05-4de1-9ba1-4e9a7aff7d41"} {"level":"info","ts":1789175406.4052088,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175406.4053154,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175406.405329,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175406.405341,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","EoS":true} {"level":"info","ts":1789175406.405367,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","header":"x-maas-username"} {"level":"info","ts":1789175406.4053774,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","header":"x-maas-subscription"} {"level":"info","ts":1789175406.4053824,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","header":"x-maas-group"} {"level":"info","ts":1789175406.4053986,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175406.4054275,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175406.4064538,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175406.4064763,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175406.4064887,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","EoS":false} {"level":"info","ts":1789175406.4065018,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175406.406529,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a","EoS":true} {"level":"info","ts":1789175406.4065387,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fd8f0a4f902a387ea2cfbca726b00112","span_id":"69d51f90c89acaac","x-request-id":"e54b9897-7a9d-4116-b6f1-5780f3101c5a"} {"level":"info","ts":1789175407.43939,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175407.4395065,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"info","ts":1789175407.4395206,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"info","ts":1789175407.4395323,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","EoS":true} {"level":"info","ts":1789175407.4395642,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","header":"x-maas-username"} {"level":"info","ts":1789175407.4395773,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","header":"x-maas-group"} {"level":"info","ts":1789175407.439582,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","header":"x-maas-subscription"} {"level":"info","ts":1789175407.4395988,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175407.4396267,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"info","ts":1789175407.4404685,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"info","ts":1789175407.44049,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"info","ts":1789175407.440509,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","EoS":false} {"level":"info","ts":1789175407.440524,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"info","ts":1789175407.4405491,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151","EoS":true} {"level":"info","ts":1789175407.4405594,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"98cb4ca58ef9a66c6b87f519fe8505e6","span_id":"5ddf8380f0f0ea4e","x-request-id":"9e6c1e74-6e59-400f-9bd4-9bd7fd341151"} {"level":"error","ts":1789175416.5708215,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789175426.5719805,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175426.902248,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175426.9023547,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"info","ts":1789175426.9023666,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"info","ts":1789175426.9023788,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","EoS":true} {"level":"info","ts":1789175426.9024036,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","header":"x-maas-username"} {"level":"info","ts":1789175426.9024136,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","header":"x-maas-group"} {"level":"info","ts":1789175426.902419,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","header":"x-maas-subscription"} {"level":"info","ts":1789175426.902438,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175426.9024637,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"info","ts":1789175426.9033594,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"info","ts":1789175426.9033813,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"info","ts":1789175426.9034023,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","EoS":false} {"level":"info","ts":1789175426.9034219,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"info","ts":1789175426.9034612,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b","EoS":true} {"level":"info","ts":1789175426.9034708,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ce7dbc4e394bf5bb573c6d2229392e56","span_id":"23e126b52bc564e0","x-request-id":"0b5fa88b-57c5-473b-a4d7-8626ab1dfb4b"} {"level":"error","ts":1789175441.5732493,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175520.6205602,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175520.620674,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"info","ts":1789175520.6206868,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"info","ts":1789175520.6207001,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","EoS":true} {"level":"info","ts":1789175520.6207275,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","header":"x-maas-group"} {"level":"info","ts":1789175520.620738,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","header":"x-maas-username"} {"level":"info","ts":1789175520.6207428,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","header":"x-maas-subscription"} {"level":"info","ts":1789175520.6207895,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789175520.6208184,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"info","ts":1789175520.6345863,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"info","ts":1789175520.6346166,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"info","ts":1789175520.63464,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","EoS":false} {"level":"info","ts":1789175520.6346543,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"info","ts":1789175520.634685,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c","EoS":true} {"level":"info","ts":1789175520.6346958,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a451fcf9b700276aab233ac8a71f6360","span_id":"fc32cc08b48f9049","x-request-id":"d708671c-1cf0-4b9e-a08c-cb5003dc2b1c"} {"level":"error","ts":1789175531.585636,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175604.9687326,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175604.9688675,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"info","ts":1789175604.968882,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"info","ts":1789175604.9688947,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","EoS":true} {"level":"info","ts":1789175604.9689286,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","header":"x-maas-username"} {"level":"info","ts":1789175604.968942,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","header":"x-maas-group"} {"level":"info","ts":1789175604.96895,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","header":"x-maas-subscription"} {"level":"info","ts":1789175604.9689674,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789175604.968993,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"info","ts":1789175604.9700103,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"info","ts":1789175604.9700358,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"info","ts":1789175604.9700592,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","EoS":false} {"level":"info","ts":1789175604.970077,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"info","ts":1789175604.9701073,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725","EoS":true} {"level":"info","ts":1789175604.9701188,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5739445975a5d21df80a6149c81c23f2","span_id":"f19188dbf40577a5","x-request-id":"9aebc4d2-4688-4364-8f6c-355ae9d85725"} {"level":"error","ts":1789175616.5956657,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175618.6349862,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175618.635088,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6350996,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6351116,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","EoS":true} {"level":"info","ts":1789175618.6351354,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","header":"x-maas-group"} {"level":"info","ts":1789175618.6351452,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","header":"x-maas-username"} {"level":"info","ts":1789175618.635151,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","header":"x-maas-subscription"} {"level":"info","ts":1789175618.6351717,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789175618.6352062,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6477773,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6478016,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6478198,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","EoS":false} {"level":"info","ts":1789175618.647838,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6478643,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","EoS":false} {"level":"info","ts":1789175618.6478755,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.6478863,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca","EoS":true} {"level":"info","ts":1789175618.647896,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ab56e1dba63dd1d6f899ff1081c41983","span_id":"171a0bf654ac3082","x-request-id":"1ba76a24-c269-42ee-b557-c30773d642ca"} {"level":"info","ts":1789175618.7888021,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175618.7889104,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.7889273,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.7889445,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","EoS":true} {"level":"info","ts":1789175618.7889702,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","header":"x-maas-group"} {"level":"info","ts":1789175618.788982,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","header":"x-maas-username"} {"level":"info","ts":1789175618.7889872,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","header":"x-maas-subscription"} {"level":"info","ts":1789175618.789003,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789175618.7890337,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.789906,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.7899272,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.7899497,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","EoS":false} {"level":"info","ts":1789175618.7899687,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.789995,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe","EoS":true} {"level":"info","ts":1789175618.7900057,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c383158cf86d3912133b4b3edb740883","span_id":"27838f7155615952","x-request-id":"202dddc3-cafc-4072-a593-d69b58dd6cfe"} {"level":"info","ts":1789175618.9291303,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175618.9292264,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"info","ts":1789175618.9292397,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"info","ts":1789175618.9292517,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","EoS":true} {"level":"info","ts":1789175618.9292727,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","header":"x-maas-subscription"} {"level":"info","ts":1789175618.929316,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","header":"x-maas-username"} {"level":"info","ts":1789175618.9293246,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","header":"x-maas-group"} {"level":"info","ts":1789175618.9293401,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789175618.9293687,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"info","ts":1789175618.9302287,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"info","ts":1789175618.930248,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"info","ts":1789175618.9302676,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","EoS":false} {"level":"info","ts":1789175618.9302816,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"info","ts":1789175618.9303083,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5","EoS":true} {"level":"info","ts":1789175618.9303184,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9c4619d554fa086475849a17bfd18dbb","span_id":"f2cd402956a1b13c","x-request-id":"5d93c67b-9ec1-4d5d-b1a4-863470847be5"} {"level":"error","ts":1789175631.596664,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789175631.7269568,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789175631.7270696,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7270849,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7271028,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","EoS":true} {"level":"info","ts":1789175631.7271366,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","header":"x-maas-group"} {"level":"info","ts":1789175631.7271495,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","header":"x-maas-username"} {"level":"info","ts":1789175631.7271578,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","header":"x-maas-subscription"} {"level":"info","ts":1789175631.7271788,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789175631.7272124,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7298915,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7299113,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7299266,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","EoS":false} {"level":"info","ts":1789175631.7299387,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7299662,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","EoS":false} {"level":"info","ts":1789175631.72998,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"} {"level":"info","ts":1789175631.7299914,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2","EoS":true} {"level":"info","ts":1789175631.7300007,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9dc4eae1b5f306202e616f2239e53994","span_id":"504002ece27c6252","x-request-id":"68c50020-eb4a-4248-b670-494ffdb65dc2"}