{"level":"info","ts":1789560867.1084316,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"3140f077-0b31-47f8-9e1b-5ab774a7f249","provider":"e2e-legacy-migration-fc03db4e","phase":"Failed"} {"level":"info","ts":1789560867.109149,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7f6db965-0699-4e7a-990f-2156d07de79f","provider":"openai","model":"e2e-legacy-migration-fc03db4e","targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789560867.1097295,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"55eb5603-2429-4911-8381-cc1abb1d03eb","error":"failed to apply ServiceEntry: serviceentries.networking.istio.io \"e2e-legacy-migration-fc03db4e\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789560867.1098888,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"2c4f9f29-0e68-441a-b8be-c9490626192c"} {"level":"info","ts":1789560867.1100032,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"2c4f9f29-0e68-441a-b8be-c9490626192c","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.1445436,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"e9f13a02-f6ea-4927-88c1-27ae10415a71","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.1445956,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"e9f13a02-f6ea-4927-88c1-27ae10415a71","modelName":"e2e-legacy-migration-fc03db4e","resolvedRefs":1} {"level":"error","ts":1789560867.1467566,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"3140f077-0b31-47f8-9e1b-5ab774a7f249","error":"ExternalModel \"e2e-legacy-migration-fc03db4e\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-fc03db4e\" is not ready (phase: Failed)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789560867.1469667,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"6d8037c3-f489-4e34-a887-6613703f0607"} {"level":"info","ts":1789560867.1470425,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"6d8037c3-f489-4e34-a887-6613703f0607","provider":"e2e-legacy-migration-fc03db4e","phase":"Failed"} {"level":"error","ts":1789560867.1696064,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"6d8037c3-f489-4e34-a887-6613703f0607","error":"ExternalModel \"e2e-legacy-migration-fc03db4e\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-fc03db4e\" is not ready (phase: Failed)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789560867.1747065,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"2c4f9f29-0e68-441a-b8be-c9490626192c","service":"e2e-legacy-migration-fc03db4e","serviceEntry":"e2e-legacy-migration-fc03db4e","destinationRule":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.1874673,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"052f165b-959f-45f8-9779-11a507849e15","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.1875522,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"052f165b-959f-45f8-9779-11a507849e15","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789560867.1875823,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"59ac48ec-0d60-450a-a4e7-dcc9cfe599ac"} {"level":"info","ts":1789560867.1876345,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"59ac48ec-0d60-450a-a4e7-dcc9cfe599ac","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.1877265,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7496077c-7c84-4c00-9188-4a78515ed931","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.1877792,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7496077c-7c84-4c00-9188-4a78515ed931","modelName":"e2e-legacy-migration-fc03db4e","resolvedRefs":1} {"level":"info","ts":1789560867.1878667,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"b4b4c6a3-41de-4c03-be00-5c108e6092a8"} {"level":"info","ts":1789560867.1879396,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"b4b4c6a3-41de-4c03-be00-5c108e6092a8","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.2093499,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"b4b4c6a3-41de-4c03-be00-5c108e6092a8","service":"e2e-legacy-migration-fc03db4e","serviceEntry":"e2e-legacy-migration-fc03db4e","destinationRule":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.216062,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"86f4c885-e1b1-41fe-8a50-d4b49d89beb5","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.2161083,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"86f4c885-e1b1-41fe-8a50-d4b49d89beb5","modelName":"e2e-legacy-migration-fc03db4e","resolvedRefs":1} {"level":"error","ts":1789560867.221575,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"59ac48ec-0d60-450a-a4e7-dcc9cfe599ac","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fc03db4e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789560867.2217007,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"59ac48ec-0d60-450a-a4e7-dcc9cfe599ac","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-fc03db4e\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789560867.2218063,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"bd237849-9b20-4e20-8a77-cd8ffd743a5f"} {"level":"info","ts":1789560867.2219071,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"bd237849-9b20-4e20-8a77-cd8ffd743a5f","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.232455,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"bd237849-9b20-4e20-8a77-cd8ffd743a5f","httpRoute":"e2e-legacy-migration-fc03db4e","providers":["e2e-legacy-migration-fc03db4e"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789560867.2392824,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"30de6d72-1fe9-4fa1-8021-f2c90ee6bd9e","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.2394097,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"30de6d72-1fe9-4fa1-8021-f2c90ee6bd9e","modelName":"e2e-legacy-migration-fc03db4e","resolvedRefs":1} {"level":"error","ts":1789560867.2407453,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"bd237849-9b20-4e20-8a77-cd8ffd743a5f","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fc03db4e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789560867.2409108,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7f4ce1c8-b3b0-496e-bd46-680632046165"} {"level":"info","ts":1789560867.2410374,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7f4ce1c8-b3b0-496e-bd46-680632046165","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.2599537,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7f4ce1c8-b3b0-496e-bd46-680632046165","httpRoute":"e2e-legacy-migration-fc03db4e","providers":["e2e-legacy-migration-fc03db4e"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789560867.2706275,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"7f4ce1c8-b3b0-496e-bd46-680632046165","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fc03db4e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789560867.2707918,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"f310d8bf-1667-4bba-af3a-687c9f29c03a"} {"level":"info","ts":1789560867.2708938,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"f310d8bf-1667-4bba-af3a-687c9f29c03a","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.2707922,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"222a7385-bdee-463f-8eea-6cc20ac2b9c9","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.2746465,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"222a7385-bdee-463f-8eea-6cc20ac2b9c9","modelName":"e2e-legacy-migration-fc03db4e","resolvedRefs":1} {"level":"info","ts":1789560867.2902725,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"f310d8bf-1667-4bba-af3a-687c9f29c03a","httpRoute":"e2e-legacy-migration-fc03db4e","providers":["e2e-legacy-migration-fc03db4e"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789560867.3717022,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"95c0f09d-cb2d-4aa4-8f98-36249c9433d3","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560867.3763542,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"95c0f09d-cb2d-4aa4-8f98-36249c9433d3","name":"e2e-legacy-migration-fc03db4e"} {"level":"info","ts":1789560867.386019,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"95c0f09d-cb2d-4aa4-8f98-36249c9433d3","provider":"openai","model":"e2e-legacy-migration-fc03db4e","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789560869.9986248,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"9834437e-519d-4e45-b576-c526e6517415","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560870.014091,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"50dce857-83cb-47c2-9b86-facae419598d"} {"level":"info","ts":1789560870.0142252,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"726c1b12-6776-4b36-b3ac-ced2301d685b","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560870.0143476,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"726c1b12-6776-4b36-b3ac-ced2301d685b","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560870.0171194,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"e28b954e-a80e-4345-9f1e-68d5e2fcdd3f","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560870.0172093,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"e28b954e-a80e-4345-9f1e-68d5e2fcdd3f","name":"e2e-legacy-migration-fc03db4e","namespace":"llm"} {"level":"info","ts":1789560870.0172887,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fc03db4e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fc03db4e","reconcileID":"ddb43e7a-9dfe-4444-91a8-5aa49e4b218f"} {"level":"info","ts":1789560870.4814363,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560870.481557,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560870.4815743,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560870.4815977,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","EoS":true} {"level":"info","ts":1789560870.4816253,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","header":"x-maas-group"} {"level":"info","ts":1789560870.4816353,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","header":"x-maas-username"} {"level":"info","ts":1789560870.4816408,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","header":"x-maas-subscription"} {"level":"info","ts":1789560870.4816554,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789560870.481666,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","modelName":"e2e-external-model"} {"level":"info","ts":1789560870.4816768,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789560870.481699,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789560870.4817152,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","authType":"apikey"} {"level":"info","ts":1789560870.4817424,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560871.4324224,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560871.4325929,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.4326265,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.4326692,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","EoS":true} {"level":"info","ts":1789560871.4327166,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","header":"x-maas-subscription"} {"level":"info","ts":1789560871.4327538,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","header":"x-maas-group"} {"level":"info","ts":1789560871.432775,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","header":"x-maas-username"} {"level":"info","ts":1789560871.4328158,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789560871.432857,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.4344776,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.434554,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.4345963,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","EoS":false} {"level":"info","ts":1789560871.4346337,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.4346888,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571","EoS":true} {"level":"info","ts":1789560871.434723,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3b98d22693fbd1fdcdc514364deeb8b2","span_id":"603a06231ea8631f","x-request-id":"22aa0ad9-7888-4891-8ab9-1551b7f4c571"} {"level":"info","ts":1789560871.4784694,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560871.4786255,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.4787035,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.4787605,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","EoS":true} {"level":"info","ts":1789560871.4788344,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","header":"x-maas-username"} {"level":"info","ts":1789560871.4788873,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","header":"x-maas-group"} {"level":"info","ts":1789560871.478915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","header":"x-maas-subscription"} {"level":"info","ts":1789560871.478972,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789560871.4790435,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.4800844,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.4801464,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.4801872,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","EoS":false} {"level":"info","ts":1789560871.4802244,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.4802754,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b","EoS":true} {"level":"info","ts":1789560871.4803696,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"073abe7eadfd781627e8aa6cc07f1215","span_id":"3229ee3b1e8e2584","x-request-id":"8232da00-c782-4174-abe4-6eec204e383b"} {"level":"info","ts":1789560871.5182955,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560871.5184574,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5184922,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5185108,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","EoS":true} {"level":"info","ts":1789560871.5185368,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","header":"x-maas-username"} {"level":"info","ts":1789560871.5185483,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","header":"x-maas-subscription"} {"level":"info","ts":1789560871.518572,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","header":"x-maas-group"} {"level":"info","ts":1789560871.518591,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789560871.5186222,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5205138,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5205715,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5206108,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","EoS":false} {"level":"info","ts":1789560871.5206451,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5206926,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334","EoS":true} {"level":"info","ts":1789560871.5207796,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c0961788742db04df220ed5580a8726c","span_id":"69379cb07672a7c6","x-request-id":"ae8072e9-a53a-463d-a92b-3a0566ef6334"} {"level":"info","ts":1789560871.5588474,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560871.5590055,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560871.5590208,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560871.5590384,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","EoS":true} {"level":"info","ts":1789560871.5590866,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","header":"x-maas-group"} {"level":"info","ts":1789560871.5590994,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","header":"x-maas-subscription"} {"level":"info","ts":1789560871.5591075,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","header":"x-maas-username"} {"level":"info","ts":1789560871.5591455,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789560871.5591776,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560871.5605526,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560871.5605752,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560871.5605931,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","EoS":false} {"level":"info","ts":1789560871.5606272,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560871.5606585,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c","EoS":true} {"level":"info","ts":1789560871.5606718,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b0bdeef48a4a55dee17bfb5827878cec","span_id":"de1d60cd1133d607","x-request-id":"de01eb56-9747-4045-b39b-0d776e53ca8c"} {"level":"info","ts":1789560873.0535374,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560873.0536,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560873.0536299,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","EoS":false} {"level":"info","ts":1789560873.0536659,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560873.0537014,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600","EoS":true} {"level":"info","ts":1789560873.053713,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad5f2b18eedeac596858053cbd3b769a","span_id":"6289edca51e59190","x-request-id":"ae636d45-4152-400d-910f-72987c0be600"} {"level":"info","ts":1789560873.1058428,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560873.1060054,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"info","ts":1789560873.1060224,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"info","ts":1789560873.106058,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","EoS":true} {"level":"info","ts":1789560873.1060927,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","header":"x-maas-subscription"} {"level":"info","ts":1789560873.106105,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","header":"x-maas-group"} {"level":"info","ts":1789560873.1061301,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","header":"x-maas-username"} {"level":"info","ts":1789560873.1061506,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789560873.106182,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"error","ts":1789560874.072465,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789560875.439857,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"info","ts":1789560875.4398952,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"info","ts":1789560875.4399195,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","EoS":false} {"level":"info","ts":1789560875.4399397,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"info","ts":1789560875.4399798,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf","EoS":true} {"level":"info","ts":1789560875.4399915,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9427dff8b47e63524b9078a72c46c540","span_id":"4e3cd1d89b1ceecb","x-request-id":"17862223-48f1-48b2-80aa-6c5455e80adf"} {"level":"info","ts":1789560875.4930792,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560875.4931831,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560875.4931953,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560875.4932067,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207","EoS":true} {"level":"info","ts":1789560875.493229,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207","header":"x-maas-subscription"} {"level":"info","ts":1789560875.4932387,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207","header":"x-maas-group"} {"level":"info","ts":1789560875.493243,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207","header":"x-maas-username"} {"level":"info","ts":1789560875.4932756,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560877.2048464,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560877.2049088,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560877.2049341,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207","EoS":false} {"level":"info","ts":1789560877.204973,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560877.2050107,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207","EoS":true} {"level":"info","ts":1789560877.2050424,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5a2a626553391bd913276a388853e6d1","span_id":"c7a6c8fa060fa1a3","x-request-id":"e19c7ab4-069b-4030-bff1-96a0eaeba207"} {"level":"info","ts":1789560878.0806026,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"904a2434-d132-4cf8-a8db-17ef5076fc03"} {"level":"info","ts":1789560878.0806155,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"aa38847b-2953-4b17-933a-a63519e5858f","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789560878.080699,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"aa38847b-2953-4b17-933a-a63519e5858f","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789560878.2329147,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"f6da567a-7713-4cb4-833a-1e16ad014830","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789560878.2331169,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"f6da567a-7713-4cb4-833a-1e16ad014830","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789560878.2330701,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"924e898b-dc97-4a17-888c-bf7661f3b01a"} {"level":"info","ts":1789560878.340108,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"e62e2bdd-2a88-4bc9-a873-97d298d42bab","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789560878.3401628,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"e62e2bdd-2a88-4bc9-a873-97d298d42bab","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789560883.9206052,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560883.9206984,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"251d4cb0f96c2cb6b0acc3503a6568af","span_id":"bcfd248af6919afb","x-request-id":"749a8ef9-a230-47c9-8e04-173331adef90"} {"level":"info","ts":1789560883.9207108,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"251d4cb0f96c2cb6b0acc3503a6568af","span_id":"bcfd248af6919afb","x-request-id":"749a8ef9-a230-47c9-8e04-173331adef90"} {"level":"info","ts":1789560883.9207227,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"251d4cb0f96c2cb6b0acc3503a6568af","span_id":"bcfd248af6919afb","x-request-id":"749a8ef9-a230-47c9-8e04-173331adef90","EoS":true} {"level":"info","ts":1789560883.9208124,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"251d4cb0f96c2cb6b0acc3503a6568af","span_id":"bcfd248af6919afb","x-request-id":"749a8ef9-a230-47c9-8e04-173331adef90"} {"level":"info","ts":1789560884.0813994,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560884.081501,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3c9ae59c85b21594c8172c5b854bfacd","span_id":"6bf3318fef91511d","x-request-id":"828ccdb2-0d30-44b3-84e5-b9b7f5ed031a"} {"level":"info","ts":1789560884.0815203,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3c9ae59c85b21594c8172c5b854bfacd","span_id":"6bf3318fef91511d","x-request-id":"828ccdb2-0d30-44b3-84e5-b9b7f5ed031a"} {"level":"info","ts":1789560884.081538,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3c9ae59c85b21594c8172c5b854bfacd","span_id":"6bf3318fef91511d","x-request-id":"828ccdb2-0d30-44b3-84e5-b9b7f5ed031a","EoS":true} {"level":"info","ts":1789560884.0816016,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3c9ae59c85b21594c8172c5b854bfacd","span_id":"6bf3318fef91511d","x-request-id":"828ccdb2-0d30-44b3-84e5-b9b7f5ed031a"} {"level":"info","ts":1789560884.2224388,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789560884.22253,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e77246295e833f8767450dccdaf829ea","span_id":"508a27c38f1b8dc6","x-request-id":"88c04e46-3259-4847-8a64-035722c7fbef"} {"level":"info","ts":1789560884.2225423,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e77246295e833f8767450dccdaf829ea","span_id":"508a27c38f1b8dc6","x-request-id":"88c04e46-3259-4847-8a64-035722c7fbef"} {"level":"info","ts":1789560884.222554,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e77246295e833f8767450dccdaf829ea","span_id":"508a27c38f1b8dc6","x-request-id":"88c04e46-3259-4847-8a64-035722c7fbef","EoS":true} {"level":"info","ts":1789560884.2225935,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e77246295e833f8767450dccdaf829ea","span_id":"508a27c38f1b8dc6","x-request-id":"88c04e46-3259-4847-8a64-035722c7fbef"} {"level":"error","ts":1789560889.074031,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789560904.0751293,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562003.0394087,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562003.0395315,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0395482,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0395622,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","EoS":true} {"level":"info","ts":1789562003.0395856,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","header":"x-maas-subscription"} {"level":"info","ts":1789562003.0395963,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","header":"x-maas-group"} {"level":"info","ts":1789562003.0396008,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","header":"x-maas-username"} {"level":"info","ts":1789562003.039616,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789562003.0396438,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0412078,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0412285,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0412455,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","EoS":false} {"level":"info","ts":1789562003.0412562,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0412765,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78","EoS":true} {"level":"info","ts":1789562003.0412865,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bb53633ac918f7fae452e9348b072f04","span_id":"1be43098d66279e0","x-request-id":"165fd1e0-d58c-40ac-bee1-4601ad4d6f78"} {"level":"info","ts":1789562003.0778832,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562003.0779588,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"info","ts":1789562003.0779712,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"info","ts":1789562003.0779817,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","EoS":true} {"level":"info","ts":1789562003.0780013,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","header":"x-maas-group"} {"level":"info","ts":1789562003.0780113,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","header":"x-maas-username"} {"level":"info","ts":1789562003.078016,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","header":"x-maas-subscription"} {"level":"info","ts":1789562003.0780296,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789562003.0780528,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"info","ts":1789562003.0789967,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"info","ts":1789562003.0790257,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"info","ts":1789562003.0790458,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","EoS":false} {"level":"info","ts":1789562003.0790653,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"info","ts":1789562003.0790982,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f","EoS":true} {"level":"info","ts":1789562003.079108,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"059498624318fa8a7c83450433cfc8a2","span_id":"d3a92a26b8ffcca8","x-request-id":"c2ec56fb-acf3-4907-9da7-2b7a8aafb54f"} {"level":"error","ts":1789562014.2119632,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562344.1915677,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562344.1916802,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.191693,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.1917093,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","EoS":true} {"level":"info","ts":1789562344.1917372,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","header":"x-maas-username"} {"level":"info","ts":1789562344.191745,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","header":"x-maas-group"} {"level":"info","ts":1789562344.1917508,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","header":"x-maas-subscription"} {"level":"info","ts":1789562344.1917648,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562344.1917942,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.2062109,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.2062984,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.2063818,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","EoS":false} {"level":"info","ts":1789562344.2064216,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.206471,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71","EoS":true} {"level":"info","ts":1789562344.2064996,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2cce5e035db662d5cf0c29552d373d8d","span_id":"ac5c7cc5ca3370e0","x-request-id":"3960eabf-0c85-4ed0-95fe-07f8547e1c71"} {"level":"info","ts":1789562344.347469,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562344.347562,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.3475752,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.3475864,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","EoS":true} {"level":"info","ts":1789562344.3476095,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","header":"x-maas-group"} {"level":"info","ts":1789562344.3476202,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","header":"x-maas-username"} {"level":"info","ts":1789562344.3476255,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","header":"x-maas-subscription"} {"level":"info","ts":1789562344.3476393,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562344.3476617,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.3483503,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.3483794,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.3483968,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","EoS":false} {"level":"info","ts":1789562344.3484116,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.3484428,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf","EoS":true} {"level":"info","ts":1789562344.348455,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3a4b5f950d7974fa7cbcee936b8bfcaf","span_id":"ededa48b1e614b76","x-request-id":"6ec98e96-b92f-4ba9-b911-981561a994cf"} {"level":"info","ts":1789562344.4821472,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562344.4822376,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.4822493,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.4822614,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","EoS":true} {"level":"info","ts":1789562344.4822812,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","header":"x-maas-username"} {"level":"info","ts":1789562344.4822912,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","header":"x-maas-group"} {"level":"info","ts":1789562344.482297,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","header":"x-maas-subscription"} {"level":"info","ts":1789562344.4823358,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562344.4823632,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.4840264,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.4840488,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.484069,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","EoS":false} {"level":"info","ts":1789562344.4840841,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.4841216,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9","EoS":true} {"level":"info","ts":1789562344.4841325,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e6b2c21b25bdc9b8b943d195f128f911","span_id":"717d55b112b29f6a","x-request-id":"a5c458ab-6b5a-4e5f-9761-31002224dda9"} {"level":"info","ts":1789562344.6223912,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562344.622495,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.6225102,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.622523,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","EoS":true} {"level":"info","ts":1789562344.6225429,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","header":"x-maas-group"} {"level":"info","ts":1789562344.622553,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","header":"x-maas-subscription"} {"level":"info","ts":1789562344.6225595,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","header":"x-maas-username"} {"level":"info","ts":1789562344.6225722,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562344.6225991,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.6232274,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.6232502,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.6232648,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","EoS":false} {"level":"info","ts":1789562344.6232762,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.623297,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548","EoS":true} {"level":"info","ts":1789562344.6233065,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0a0fb48fe067e80e1d2c08788d68a2f1","span_id":"a0b58c7c4b763896","x-request-id":"dec5692f-f6c5-4e5c-aaa5-77ef10453548"} {"level":"info","ts":1789562344.75876,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562344.7588553,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562344.7588701,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562344.7588832,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","EoS":true} {"level":"info","ts":1789562344.7589123,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","header":"x-maas-username"} {"level":"info","ts":1789562344.758923,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","header":"x-maas-group"} {"level":"info","ts":1789562344.758929,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","header":"x-maas-subscription"} {"level":"info","ts":1789562344.7589421,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562344.758965,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562344.7596116,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562344.7596312,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562344.7596455,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","EoS":false} {"level":"info","ts":1789562344.7596593,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562344.759686,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac","EoS":true} {"level":"info","ts":1789562344.7597,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ce04527cd877c82c7edaafe0255831f","span_id":"a9d491b8ad1f227f","x-request-id":"72cca5cf-e88e-469d-87d8-2f8acd0032ac"} {"level":"info","ts":1789562353.6602905,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562353.6603975,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.6604111,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.6604264,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","EoS":true} {"level":"info","ts":1789562353.660454,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","header":"x-maas-group"} {"level":"info","ts":1789562353.6604657,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","header":"x-maas-subscription"} {"level":"info","ts":1789562353.6604702,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","header":"x-maas-username"} {"level":"info","ts":1789562353.6604838,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562353.6605136,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.6611614,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.6611812,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.6611953,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","EoS":false} {"level":"info","ts":1789562353.6612065,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.661226,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67","EoS":true} {"level":"info","ts":1789562353.6612349,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c2f78f254c9498922437cf15a3475cb","span_id":"8df73845b6773a20","x-request-id":"65c3abd3-cf84-4911-ad8b-d8bd8ac66b67"} {"level":"info","ts":1789562353.7618544,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562353.7619426,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268"} {"level":"info","ts":1789562353.7624812,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268"} {"level":"info","ts":1789562353.7625015,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268"} {"level":"info","ts":1789562353.7625153,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268","EoS":false} {"level":"info","ts":1789562353.762528,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268"} {"level":"info","ts":1789562353.7625477,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268","EoS":true} {"level":"info","ts":1789562353.7625573,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ce26e95cf1e5956308235abe94b9eba0","span_id":"48441978deb0f661","x-request-id":"6c60e76c-2f66-909b-b41a-fab018795268"} {"level":"error","ts":1789562354.2568262,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562354.514554,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562354.5146475,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562354.5146613,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562354.5146728,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","EoS":true} {"level":"info","ts":1789562354.5146942,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","header":"x-maas-username"} {"level":"info","ts":1789562354.514704,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","header":"x-maas-subscription"} {"level":"info","ts":1789562354.5147092,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","header":"x-maas-group"} {"level":"info","ts":1789562354.5147235,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562354.5147505,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562354.516491,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562354.5165071,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562354.51652,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","EoS":false} {"level":"info","ts":1789562354.5165303,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562354.516553,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a","EoS":true} {"level":"info","ts":1789562354.516562,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"96f29c3fd3f6dabc1176d0ac6e42c79b","span_id":"31312d7da982c3d7","x-request-id":"7b34dc61-f301-4e43-8d01-8aabe44b5e9a"} {"level":"info","ts":1789562355.446435,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562355.446535,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562355.4465497,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562355.4465625,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","EoS":true} {"level":"info","ts":1789562355.446583,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","header":"x-maas-group"} {"level":"info","ts":1789562355.446593,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","header":"x-maas-username"} {"level":"info","ts":1789562355.4465983,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","header":"x-maas-subscription"} {"level":"info","ts":1789562355.4466121,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562355.4466388,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562355.4474897,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562355.4475095,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562355.447524,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","EoS":false} {"level":"info","ts":1789562355.4475513,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562355.447582,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589","EoS":true} {"level":"info","ts":1789562355.4475932,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2b5d342c18ecf4b272b5a33d16dfcc84","span_id":"8ae187b70b7b363e","x-request-id":"a638dfc4-03bc-46b6-b5b2-65668c221589"} {"level":"info","ts":1789562356.16936,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562356.1695507,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562356.169573,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562356.1695905,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","EoS":true} {"level":"info","ts":1789562356.1696208,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","header":"x-maas-username"} {"level":"info","ts":1789562356.1696324,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","header":"x-maas-group"} {"level":"info","ts":1789562356.1696408,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","header":"x-maas-subscription"} {"level":"info","ts":1789562356.1696582,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562356.1697292,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562356.1707838,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562356.1709108,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562356.1709604,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","EoS":false} {"level":"info","ts":1789562356.1710129,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562356.1710684,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8","EoS":true} {"level":"info","ts":1789562356.1711528,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c89a44a9f3bb65201d3fb6f565076258","span_id":"edbd700f3f28f8e9","x-request-id":"5d43e748-ccf6-40fd-82a6-c523d447d6e8"} {"level":"info","ts":1789562357.2320924,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562357.2321875,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"info","ts":1789562357.2322004,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"info","ts":1789562357.2322118,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","EoS":true} {"level":"info","ts":1789562357.2322333,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","header":"x-maas-username"} {"level":"info","ts":1789562357.232243,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","header":"x-maas-subscription"} {"level":"info","ts":1789562357.2322488,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","header":"x-maas-group"} {"level":"info","ts":1789562357.2322624,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562357.2322872,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"info","ts":1789562357.2332313,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"info","ts":1789562357.2332516,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"info","ts":1789562357.2332685,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","EoS":false} {"level":"info","ts":1789562357.2332897,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"info","ts":1789562357.2333431,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01","EoS":true} {"level":"info","ts":1789562357.2333558,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4476d18d26b35d61b69ea64cae82fba7","span_id":"830cd6141f998802","x-request-id":"d3612ad3-bca5-41f1-8fa2-e40300032c01"} {"level":"error","ts":1789562364.2578235,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789562374.2586315,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562374.6164668,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562374.6165733,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"info","ts":1789562374.6165864,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"info","ts":1789562374.6166039,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","EoS":true} {"level":"info","ts":1789562374.61664,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","header":"x-maas-username"} {"level":"info","ts":1789562374.616652,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","header":"x-maas-subscription"} {"level":"info","ts":1789562374.6166599,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","header":"x-maas-group"} {"level":"info","ts":1789562374.6166804,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562374.6167133,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"info","ts":1789562374.617587,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"info","ts":1789562374.617609,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"info","ts":1789562374.6176238,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","EoS":false} {"level":"info","ts":1789562374.6176362,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"info","ts":1789562374.6176577,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23","EoS":true} {"level":"info","ts":1789562374.6176686,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"56ff7c85a3f76f516a0a2af2e8a224b3","span_id":"6fdf73246365a5dc","x-request-id":"36a10e46-abef-428d-98c8-84836a27ac23"} {"level":"error","ts":1789562384.2597187,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789562394.2604547,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789562404.260724,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789562414.2611287,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789562424.2613811,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562463.8804057,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562463.8805125,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"info","ts":1789562463.8805292,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"info","ts":1789562463.8805425,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","EoS":true} {"level":"info","ts":1789562463.8805625,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","header":"x-maas-group"} {"level":"info","ts":1789562463.8805704,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","header":"x-maas-username"} {"level":"info","ts":1789562463.8805764,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","header":"x-maas-subscription"} {"level":"info","ts":1789562463.8805897,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789562463.880614,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"info","ts":1789562463.8935933,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"info","ts":1789562463.8936162,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"info","ts":1789562463.8936315,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","EoS":false} {"level":"info","ts":1789562463.8936443,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"info","ts":1789562463.89367,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab","EoS":true} {"level":"info","ts":1789562463.8936818,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"18e5212812fd55823fbde49373580dfa","span_id":"61207e200514f46b","x-request-id":"cd3c20bf-99d4-40dd-8d13-f9d39e1e99ab"} {"level":"error","ts":1789562474.2669158,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562534.2610872,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562534.2611904,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"info","ts":1789562534.2612047,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"info","ts":1789562534.2612164,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","EoS":true} {"level":"info","ts":1789562534.261237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","header":"x-maas-username"} {"level":"info","ts":1789562534.2612474,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","header":"x-maas-subscription"} {"level":"info","ts":1789562534.2612524,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","header":"x-maas-group"} {"level":"info","ts":1789562534.2612655,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789562534.2612922,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"info","ts":1789562534.2619967,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"info","ts":1789562534.2620182,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"info","ts":1789562534.2620344,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","EoS":false} {"level":"info","ts":1789562534.262053,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"info","ts":1789562534.2620795,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732","EoS":true} {"level":"info","ts":1789562534.2620895,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"17769d40d193c15da011b63ed26a9f10","span_id":"8f1a2882a1ed5acc","x-request-id":"91ef9e3c-16f6-47a1-9ac2-bef09b0fa732"} {"level":"error","ts":1789562544.2745187,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562548.2904332,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562548.290542,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.290556,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.2905674,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","EoS":true} {"level":"info","ts":1789562548.2905943,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","header":"x-maas-group"} {"level":"info","ts":1789562548.290605,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","header":"x-maas-username"} {"level":"info","ts":1789562548.2906098,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","header":"x-maas-subscription"} {"level":"info","ts":1789562548.2906249,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789562548.2906528,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.3037112,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.303743,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.3037708,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","EoS":false} {"level":"info","ts":1789562548.303786,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.303822,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","EoS":false} {"level":"info","ts":1789562548.3038328,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.3038466,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","EoS":false} {"level":"info","ts":1789562548.303855,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.3038664,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583","EoS":true} {"level":"info","ts":1789562548.303878,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1663066e5feacb874f92744df0dd715b","span_id":"7abcc2a6563feeac","x-request-id":"58d225cd-92eb-41fa-b13d-2598bfc87583"} {"level":"info","ts":1789562548.4404407,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562548.440521,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.4405324,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.4405425,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","EoS":true} {"level":"info","ts":1789562548.4405599,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","header":"x-maas-username"} {"level":"info","ts":1789562548.4405696,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","header":"x-maas-group"} {"level":"info","ts":1789562548.4405744,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","header":"x-maas-subscription"} {"level":"info","ts":1789562548.4405878,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789562548.4406137,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.441484,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.4415135,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.4415348,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","EoS":false} {"level":"info","ts":1789562548.4415548,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.4415803,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","EoS":false} {"level":"info","ts":1789562548.4415948,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.4416132,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693","EoS":true} {"level":"info","ts":1789562548.4416227,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c94f2966496427b27792f172baa0544d","span_id":"d989ca306dee8eea","x-request-id":"2def7206-f002-4c76-b7c0-356e36aa5693"} {"level":"info","ts":1789562548.5761187,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562548.576212,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.576225,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.5762367,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","EoS":true} {"level":"info","ts":1789562548.5762594,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","header":"x-maas-group"} {"level":"info","ts":1789562548.5762749,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","header":"x-maas-username"} {"level":"info","ts":1789562548.5762806,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","header":"x-maas-subscription"} {"level":"info","ts":1789562548.576294,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789562548.576336,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.5784848,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.5785108,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.5785317,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","EoS":false} {"level":"info","ts":1789562548.57857,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.5786018,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","EoS":false} {"level":"info","ts":1789562548.5786128,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.5786247,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","EoS":false} {"level":"info","ts":1789562548.578636,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"info","ts":1789562548.578648,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f","EoS":true} {"level":"info","ts":1789562548.578654,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"28c07044ab35aafe920670738a49e48d","span_id":"21fa0ce7bd5f7a1b","x-request-id":"43faf810-be53-47ec-b545-52dc93d0f14f"} {"level":"error","ts":1789562559.2750955,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789562561.5425107,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789562561.5426416,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"info","ts":1789562561.5426667,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"info","ts":1789562561.542683,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","EoS":true} {"level":"info","ts":1789562561.5427058,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","header":"x-maas-username"} {"level":"info","ts":1789562561.5427144,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","header":"x-maas-subscription"} {"level":"info","ts":1789562561.5427208,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","header":"x-maas-group"} {"level":"info","ts":1789562561.5427399,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789562561.5427766,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"info","ts":1789562561.5450118,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"info","ts":1789562561.5450306,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"info","ts":1789562561.5450473,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","EoS":false} {"level":"info","ts":1789562561.545059,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"info","ts":1789562561.5450833,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8","EoS":true} {"level":"info","ts":1789562561.545095,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7195ec9c69ebdccefb364d316b908db0","span_id":"ecfcd771d7d990e1","x-request-id":"2514da79-52c2-4e0f-ba0d-a93b9f8bbfb8"} {"level":"error","ts":1789562574.2754617,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}