{"level":"info","ts":1789178304.1516495,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"bfdcad6a-9171-40fc-afc4-3e4b8435c4d1","modelName":"e2e-legacy-migration-6bf56fa4","resolvedRefs":1} {"level":"error","ts":1789178304.1516578,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"6763af92-72d9-4b44-beaf-330c9bdf783d","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-6bf56fa4\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789178304.151748,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"6763af92-72d9-4b44-beaf-330c9bdf783d","error":"failed to apply Service: services \"e2e-legacy-migration-6bf56fa4\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.1518152,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"995940a1-793f-4292-a235-dbe1a102a55e"} {"level":"info","ts":1789178304.1518667,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"995940a1-793f-4292-a235-dbe1a102a55e","name":"e2e-legacy-migration-6bf56fa4"} {"level":"error","ts":1789178304.1544597,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"e4962e45-11a7-4d5d-8e3a-50cbee35fe0c","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-6bf56fa4\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789178304.1544986,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"e4962e45-11a7-4d5d-8e3a-50cbee35fe0c","error":"ExternalModel \"e2e-legacy-migration-6bf56fa4\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-6bf56fa4\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.1545484,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"0c4dac10-7048-4d45-b4d0-57528c1214ac"} {"level":"info","ts":1789178304.15457,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"0c4dac10-7048-4d45-b4d0-57528c1214ac","provider":"e2e-legacy-migration-6bf56fa4","phase":"Pending"} {"level":"error","ts":1789178304.1587167,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"0c4dac10-7048-4d45-b4d0-57528c1214ac","error":"ExternalModel \"e2e-legacy-migration-6bf56fa4\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-6bf56fa4\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.161766,"caller":"externalprovider/reconciler.go:210","msg":"Creating resource","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"995940a1-793f-4292-a235-dbe1a102a55e","kind":"DestinationRule","name":"e2e-legacy-migration-6bf56fa4"} {"level":"info","ts":1789178304.1698475,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"c8428abf-09c9-44e2-b7b3-d77870003d3a"} {"level":"info","ts":1789178304.1698966,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"c8428abf-09c9-44e2-b7b3-d77870003d3a","provider":"e2e-legacy-migration-6bf56fa4","phase":"Pending"} {"level":"info","ts":1789178304.1713257,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"5d6382b9-23c9-4cc0-87ab-78dbbfa308ff","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178304.1718173,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"5d6382b9-23c9-4cc0-87ab-78dbbfa308ff","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789178304.1717534,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"80b99921-3e2d-481c-98b4-85595740921c","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178304.1718576,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"80b99921-3e2d-481c-98b4-85595740921c","modelName":"e2e-legacy-migration-6bf56fa4","resolvedRefs":1} {"level":"error","ts":1789178304.1777174,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"995940a1-793f-4292-a235-dbe1a102a55e","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-6bf56fa4\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789178304.177759,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"995940a1-793f-4292-a235-dbe1a102a55e","error":"failed to apply DestinationRule: destinationrules.networking.istio.io \"e2e-legacy-migration-6bf56fa4\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.1778202,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"a86a8683-48c0-4a07-9101-e0589500c704"} {"level":"info","ts":1789178304.1779072,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"a86a8683-48c0-4a07-9101-e0589500c704","name":"e2e-legacy-migration-6bf56fa4"} {"level":"error","ts":1789178304.178027,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"c8428abf-09c9-44e2-b7b3-d77870003d3a","error":"ExternalModel \"e2e-legacy-migration-6bf56fa4\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-6bf56fa4\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.1781294,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"95550df3-5508-42f1-8d8f-56bc87c72111"} {"level":"info","ts":1789178304.1781805,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"95550df3-5508-42f1-8d8f-56bc87c72111","name":"e2e-legacy-migration-6bf56fa4"} {"level":"info","ts":1789178304.1912591,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"3f513c2f-e84c-41ea-8404-437bec5fcc30","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178304.1913865,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"3f513c2f-e84c-41ea-8404-437bec5fcc30","modelName":"e2e-legacy-migration-6bf56fa4","resolvedRefs":1} {"level":"info","ts":1789178304.205301,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"a86a8683-48c0-4a07-9101-e0589500c704","service":"e2e-legacy-migration-6bf56fa4","serviceEntry":"e2e-legacy-migration-6bf56fa4","destinationRule":"e2e-legacy-migration-6bf56fa4"} {"level":"error","ts":1789178304.2057562,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"95550df3-5508-42f1-8d8f-56bc87c72111","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-6bf56fa4\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789178304.2058442,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"95550df3-5508-42f1-8d8f-56bc87c72111","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-6bf56fa4\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.2059262,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"e69a713a-30b1-40a8-8967-252b5cd9a977"} {"level":"info","ts":1789178304.2060065,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"e69a713a-30b1-40a8-8967-252b5cd9a977","name":"e2e-legacy-migration-6bf56fa4"} {"level":"info","ts":1789178304.2094796,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178304.2095866,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.2095995,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.209615,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","EoS":true} {"level":"info","ts":1789178304.2096415,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","header":"x-maas-username"} {"level":"info","ts":1789178304.2096524,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","header":"x-maas-subscription"} {"level":"info","ts":1789178304.2096598,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","header":"x-maas-group"} {"level":"info","ts":1789178304.2096753,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789178304.2097008,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.2112916,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"f5473423-85ca-4f67-9dc6-b7484ddaeac5","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178304.2113278,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"f5473423-85ca-4f67-9dc6-b7484ddaeac5","modelName":"e2e-legacy-migration-6bf56fa4","resolvedRefs":1} {"level":"info","ts":1789178304.2114244,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.2114336,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.2114508,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","EoS":false} {"level":"info","ts":1789178304.2114663,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.2114935,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770","EoS":true} {"level":"info","ts":1789178304.2115042,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"98ccb2a023e8d4fb224dadedcfc1c06c","span_id":"eb9c1f6e6f5938bf","x-request-id":"6cb278d2-d507-43cf-9786-9e70289ca770"} {"level":"info","ts":1789178304.2181873,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"e69a713a-30b1-40a8-8967-252b5cd9a977","httpRoute":"e2e-legacy-migration-6bf56fa4","providers":["e2e-legacy-migration-6bf56fa4"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789178304.2293406,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"e69a713a-30b1-40a8-8967-252b5cd9a977","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-6bf56fa4\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.2294555,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"00d6dc77-71e1-4814-a839-30af5218404b"} {"level":"info","ts":1789178304.2295203,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"00d6dc77-71e1-4814-a839-30af5218404b","name":"e2e-legacy-migration-6bf56fa4"} {"level":"info","ts":1789178304.2344553,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"fcff8b26-5d83-4501-95f4-b4d1688fa07f","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178304.2344944,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"fcff8b26-5d83-4501-95f4-b4d1688fa07f","modelName":"e2e-legacy-migration-6bf56fa4","resolvedRefs":1} {"level":"info","ts":1789178304.2389615,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"00d6dc77-71e1-4814-a839-30af5218404b","httpRoute":"e2e-legacy-migration-6bf56fa4","providers":["e2e-legacy-migration-6bf56fa4"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789178304.2433183,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"00d6dc77-71e1-4814-a839-30af5218404b","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-6bf56fa4\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789178304.2434888,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"06ea397a-0b86-4d39-a607-acaa54ef3961"} {"level":"info","ts":1789178304.243592,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"06ea397a-0b86-4d39-a607-acaa54ef3961","name":"e2e-legacy-migration-6bf56fa4"} {"level":"info","ts":1789178304.2516418,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"06ea397a-0b86-4d39-a607-acaa54ef3961","httpRoute":"e2e-legacy-migration-6bf56fa4","providers":["e2e-legacy-migration-6bf56fa4"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789178304.2560847,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178304.2562363,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.2562509,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.2562637,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","EoS":true} {"level":"info","ts":1789178304.256285,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","header":"x-maas-group"} {"level":"info","ts":1789178304.2562983,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","header":"x-maas-username"} {"level":"info","ts":1789178304.2563035,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","header":"x-maas-subscription"} {"level":"info","ts":1789178304.2563162,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789178304.256345,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.257363,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.2573903,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.257414,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","EoS":false} {"level":"info","ts":1789178304.2574303,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.2574646,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392","EoS":true} {"level":"info","ts":1789178304.2574763,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a37d671d91cdd8d6ffe9910dbd301111","span_id":"d9cd56a9aaae2991","x-request-id":"92bbc225-a5e7-434e-90f9-75270fd38392"} {"level":"info","ts":1789178304.3112998,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178304.3114266,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.3114426,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.3114564,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","EoS":true} {"level":"info","ts":1789178304.3114886,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","header":"x-maas-subscription"} {"level":"info","ts":1789178304.3115041,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","header":"x-maas-username"} {"level":"info","ts":1789178304.3115091,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","header":"x-maas-group"} {"level":"info","ts":1789178304.3115246,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789178304.3115513,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.3124504,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.3124704,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.312493,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","EoS":false} {"level":"info","ts":1789178304.312504,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.3125284,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81","EoS":true} {"level":"info","ts":1789178304.3125412,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3d783766257d4c53761a07bf88970750","span_id":"5e4d4728defef1bb","x-request-id":"d3e955e3-b495-4938-a1bb-2d90fdbffb81"} {"level":"info","ts":1789178304.3530624,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178304.3532073,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.353223,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.3532398,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","EoS":true} {"level":"info","ts":1789178304.353272,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","header":"x-maas-username"} {"level":"info","ts":1789178304.3532853,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","header":"x-maas-subscription"} {"level":"info","ts":1789178304.3532922,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","header":"x-maas-group"} {"level":"info","ts":1789178304.353312,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789178304.3533459,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.3542733,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.3542986,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.3543155,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","EoS":false} {"level":"info","ts":1789178304.3543332,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.354369,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75","EoS":true} {"level":"info","ts":1789178304.354383,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"44323dab8df3736b73ef9f05ec3d2ada","span_id":"35ddf33a23c4325a","x-request-id":"7e7382ba-ffea-4613-b46e-c7449ab20d75"} {"level":"info","ts":1789178304.4807568,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"4e48a488-f0ac-4aef-b88d-ec9e3ca6cced","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178304.493465,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"4e48a488-f0ac-4aef-b88d-ec9e3ca6cced","name":"e2e-legacy-migration-6bf56fa4"} {"level":"info","ts":1789178304.5011477,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"4e48a488-f0ac-4aef-b88d-ec9e3ca6cced","provider":"openai","model":"e2e-legacy-migration-6bf56fa4","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789178307.1385458,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"a5a8a563-c380-4a07-bdbb-1a69b4ab49fb","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178307.1483355,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"5b20340a-8b5e-42f2-b53a-1d6e7092773b","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178307.14852,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"5b20340a-8b5e-42f2-b53a-1d6e7092773b","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178307.1484728,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"aac8bc3f-3989-4fd9-b6eb-5f7f26a01934"} {"level":"info","ts":1789178307.152659,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"21e75289-6c34-47ac-b854-7b2e7cb73be3"} {"level":"info","ts":1789178307.152724,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"426deaa2-9577-4473-8ec5-f6e10b4656b2","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178307.1527734,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-6bf56fa4","reconcileID":"426deaa2-9577-4473-8ec5-f6e10b4656b2","name":"e2e-legacy-migration-6bf56fa4","namespace":"llm"} {"level":"info","ts":1789178307.645285,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178307.645391,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.6454065,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.6454248,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","EoS":true} {"level":"info","ts":1789178307.6454618,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","header":"x-maas-subscription"} {"level":"info","ts":1789178307.6454754,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","header":"x-maas-username"} {"level":"info","ts":1789178307.6454835,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","header":"x-maas-group"} {"level":"info","ts":1789178307.645504,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789178307.6455157,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","modelName":"e2e-external-model"} {"level":"info","ts":1789178307.6455348,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789178307.645565,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789178307.6455898,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","authType":"apikey"} {"level":"info","ts":1789178307.6456175,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.7191806,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.7192154,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.7192407,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","EoS":false} {"level":"info","ts":1789178307.7192602,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.7192953,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2","EoS":true} {"level":"info","ts":1789178307.719307,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad88cca98c0ba18f16c319c9f7ade83b","span_id":"69cae7ee97881afd","x-request-id":"e8b96d07-6c1a-4d4f-bb7d-2d61188f65a2"} {"level":"info","ts":1789178307.766703,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178307.766824,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.7668447,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.766863,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","EoS":true} {"level":"info","ts":1789178307.7668953,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","header":"x-maas-username"} {"level":"info","ts":1789178307.7669091,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","header":"x-maas-subscription"} {"level":"info","ts":1789178307.7669175,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","header":"x-maas-group"} {"level":"info","ts":1789178307.7669356,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789178307.7669718,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.814668,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.814704,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.814726,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","EoS":false} {"level":"info","ts":1789178307.8147428,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.814778,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc","EoS":true} {"level":"info","ts":1789178307.8147912,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fcab7fffe6feb703fa58bd27d954d485","span_id":"a614c13c746a0ab7","x-request-id":"0c35ae62-4054-4b2f-9aba-58822b1f99cc"} {"level":"info","ts":1789178307.864031,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178307.864152,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178307.8642,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178307.8642476,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f","EoS":true} {"level":"info","ts":1789178307.8642964,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f","header":"x-maas-username"} {"level":"info","ts":1789178307.8643298,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f","header":"x-maas-subscription"} {"level":"info","ts":1789178307.8643508,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f","header":"x-maas-group"} {"level":"info","ts":1789178307.8643956,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178307.9013844,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178307.9015503,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178307.9016252,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f","EoS":false} {"level":"info","ts":1789178307.9016786,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178307.9017544,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f","EoS":true} {"level":"info","ts":1789178307.9018023,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7713fb02fc6b988a0b3d4ce9dd1cc939","span_id":"11c1c0391b0b68e5","x-request-id":"9f5e4e67-29f3-4983-a111-aa5e3a7b007f"} {"level":"info","ts":1789178308.8421364,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"3132e604-62de-4c69-b183-9e4d48206013"} {"level":"info","ts":1789178308.8421407,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"5e940087-8b8b-4b45-90ed-4b69ed90f9d7","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789178308.8422995,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"5e940087-8b8b-4b45-90ed-4b69ed90f9d7","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789178308.9986637,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"85baa664-43c8-4343-bb3c-9c2fd469afc4","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789178308.9987142,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"85baa664-43c8-4343-bb3c-9c2fd469afc4","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789178308.9988077,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"9257f19d-0c85-4205-81dc-e7631be5874d"} {"level":"info","ts":1789178309.1235235,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"68829a20-184e-486a-adda-056ca546c5dd","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789178309.1236951,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"68829a20-184e-486a-adda-056ca546c5dd","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789178312.227439,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789178318.7596228,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178318.7597206,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fef8c0847abb9ac55660d8ff31d2c4c4","span_id":"42aa85afbfb35249","x-request-id":"577cddfb-1e70-46ec-be1f-fa9acd492963"} {"level":"info","ts":1789178318.7597368,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fef8c0847abb9ac55660d8ff31d2c4c4","span_id":"42aa85afbfb35249","x-request-id":"577cddfb-1e70-46ec-be1f-fa9acd492963"} {"level":"info","ts":1789178318.7597544,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fef8c0847abb9ac55660d8ff31d2c4c4","span_id":"42aa85afbfb35249","x-request-id":"577cddfb-1e70-46ec-be1f-fa9acd492963","EoS":true} {"level":"info","ts":1789178318.7598145,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fef8c0847abb9ac55660d8ff31d2c4c4","span_id":"42aa85afbfb35249","x-request-id":"577cddfb-1e70-46ec-be1f-fa9acd492963"} {"level":"info","ts":1789178318.9277937,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178318.9278734,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"86e66e07dfbde56a9423515e1f6b1af1","span_id":"5c6bdc46756f9c4e","x-request-id":"14eae9e8-9404-4164-adb9-65d3180d149a"} {"level":"info","ts":1789178318.9278839,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"86e66e07dfbde56a9423515e1f6b1af1","span_id":"5c6bdc46756f9c4e","x-request-id":"14eae9e8-9404-4164-adb9-65d3180d149a"} {"level":"info","ts":1789178318.9278963,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"86e66e07dfbde56a9423515e1f6b1af1","span_id":"5c6bdc46756f9c4e","x-request-id":"14eae9e8-9404-4164-adb9-65d3180d149a","EoS":true} {"level":"info","ts":1789178318.9279368,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"86e66e07dfbde56a9423515e1f6b1af1","span_id":"5c6bdc46756f9c4e","x-request-id":"14eae9e8-9404-4164-adb9-65d3180d149a"} {"level":"info","ts":1789178319.1098566,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789178319.109945,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ce787f871d17d326768e80f530e95b1b","span_id":"114e44c15c692b2f","x-request-id":"c5fade64-8b5a-4fbb-88b0-e10e8f73fc16"} {"level":"info","ts":1789178319.109959,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ce787f871d17d326768e80f530e95b1b","span_id":"114e44c15c692b2f","x-request-id":"c5fade64-8b5a-4fbb-88b0-e10e8f73fc16"} {"level":"info","ts":1789178319.10997,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ce787f871d17d326768e80f530e95b1b","span_id":"114e44c15c692b2f","x-request-id":"c5fade64-8b5a-4fbb-88b0-e10e8f73fc16","EoS":true} {"level":"info","ts":1789178319.1100101,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ce787f871d17d326768e80f530e95b1b","span_id":"114e44c15c692b2f","x-request-id":"c5fade64-8b5a-4fbb-88b0-e10e8f73fc16"} {"level":"error","ts":1789178322.2283285,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789178337.2294323,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179392.5513964,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179392.5515175,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.5515378,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.5515583,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","EoS":true} {"level":"info","ts":1789179392.5515897,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","header":"x-maas-subscription"} {"level":"info","ts":1789179392.5515988,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","header":"x-maas-group"} {"level":"info","ts":1789179392.551604,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","header":"x-maas-username"} {"level":"info","ts":1789179392.551618,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789179392.5516436,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.5535142,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.5535417,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.553562,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","EoS":false} {"level":"info","ts":1789179392.5535789,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.5536149,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42","EoS":true} {"level":"info","ts":1789179392.5536242,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ec84d47ab90f79a32c1b8e09619fac96","span_id":"0632ae057ca4cb29","x-request-id":"9cb21fc2-98fd-4fe2-a1c6-fb7e40cc1e42"} {"level":"info","ts":1789179392.5931535,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179392.5932305,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"info","ts":1789179392.593241,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"info","ts":1789179392.5932512,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","EoS":true} {"level":"info","ts":1789179392.5932715,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","header":"x-maas-group"} {"level":"info","ts":1789179392.5932815,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","header":"x-maas-subscription"} {"level":"info","ts":1789179392.593287,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","header":"x-maas-username"} {"level":"info","ts":1789179392.593302,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789179392.5933306,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"info","ts":1789179392.5942385,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"info","ts":1789179392.5942554,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"info","ts":1789179392.5942705,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","EoS":false} {"level":"info","ts":1789179392.5942843,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"info","ts":1789179392.594312,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5","EoS":true} {"level":"info","ts":1789179392.5943222,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ebb14a27bb28bd3915d3f78cc07708ea","span_id":"a6151a50ce13e72f","x-request-id":"de5fda45-63c6-4734-88eb-3f86805e81b5"} {"level":"error","ts":1789179407.3636115,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179723.9907272,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179723.990814,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179723.9908254,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179723.9908366,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","EoS":true} {"level":"info","ts":1789179723.9908562,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","header":"x-maas-group"} {"level":"info","ts":1789179723.9908657,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","header":"x-maas-username"} {"level":"info","ts":1789179723.9908707,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","header":"x-maas-subscription"} {"level":"info","ts":1789179723.990884,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179723.9909077,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179724.0031602,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179724.0031807,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179724.0031996,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","EoS":false} {"level":"info","ts":1789179724.003212,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179724.0032318,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b","EoS":true} {"level":"info","ts":1789179724.0032384,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6a8928044597f737a2ff3ff2f6d8979a","span_id":"8be4330c3b435cec","x-request-id":"2fc71405-0ed8-4a52-8990-32b0e98f5a3b"} {"level":"info","ts":1789179724.1443508,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179724.1444342,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.144446,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.1444566,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","EoS":true} {"level":"info","ts":1789179724.144477,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","header":"x-maas-subscription"} {"level":"info","ts":1789179724.1444864,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","header":"x-maas-group"} {"level":"info","ts":1789179724.1444988,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","header":"x-maas-username"} {"level":"info","ts":1789179724.1445127,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179724.1445363,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.1451526,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.1451714,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.1451852,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","EoS":false} {"level":"info","ts":1789179724.1451998,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.1452231,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32","EoS":true} {"level":"info","ts":1789179724.1452327,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"763fb7ccc445d53dc83da50b6817d40f","span_id":"ea247ed57bf8eff3","x-request-id":"954a7140-9fff-47e7-8d97-574d08202e32"} {"level":"info","ts":1789179724.2799258,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179724.2800167,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.28003,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.2800689,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","EoS":true} {"level":"info","ts":1789179724.2800982,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","header":"x-maas-group"} {"level":"info","ts":1789179724.2801108,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","header":"x-maas-subscription"} {"level":"info","ts":1789179724.2801204,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","header":"x-maas-username"} {"level":"info","ts":1789179724.2801414,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179724.2801738,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.280774,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.2807915,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.2808065,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","EoS":false} {"level":"info","ts":1789179724.2808223,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.2808409,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea","EoS":true} {"level":"info","ts":1789179724.280851,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"aa6ebbd6ff244c2b5c5e09c9e28ecde5","span_id":"552a29ead1917056","x-request-id":"1c1948b5-85cf-4f7e-af34-ddfaac42a7ea"} {"level":"info","ts":1789179724.418682,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179724.4187818,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.4187953,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.4188132,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","EoS":true} {"level":"info","ts":1789179724.418844,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","header":"x-maas-group"} {"level":"info","ts":1789179724.418855,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","header":"x-maas-username"} {"level":"info","ts":1789179724.418864,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","header":"x-maas-subscription"} {"level":"info","ts":1789179724.418884,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179724.4189143,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.4204276,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.4204457,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.4204574,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","EoS":false} {"level":"info","ts":1789179724.4204707,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.4205015,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4","EoS":true} {"level":"info","ts":1789179724.4205127,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"435ccda33cbcb54a5e1e6aedf87b889a","span_id":"a21cd7a07fd53db1","x-request-id":"69ac6b1d-7d8e-4400-b2d1-18ef924995b4"} {"level":"info","ts":1789179724.5570612,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179724.5571454,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"info","ts":1789179724.5571582,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"info","ts":1789179724.557169,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","EoS":true} {"level":"info","ts":1789179724.5571866,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","header":"x-maas-group"} {"level":"info","ts":1789179724.5571969,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","header":"x-maas-username"} {"level":"info","ts":1789179724.5572019,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","header":"x-maas-subscription"} {"level":"info","ts":1789179724.5572183,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179724.5572412,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"info","ts":1789179724.5577872,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"info","ts":1789179724.557804,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"info","ts":1789179724.5578136,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","EoS":false} {"level":"info","ts":1789179724.5578246,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"info","ts":1789179724.5578463,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae","EoS":true} {"level":"info","ts":1789179724.557855,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2400674664a489978c355cea23e601cb","span_id":"63659cdf48d66d49","x-request-id":"163f86d3-0302-43a7-8890-b625108884ae"} {"level":"error","ts":1789179737.3988247,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179739.6629584,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179739.6630802,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.6630955,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.6631153,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","EoS":true} {"level":"info","ts":1789179739.663145,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","header":"x-maas-group"} {"level":"info","ts":1789179739.6631572,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","header":"x-maas-username"} {"level":"info","ts":1789179739.6631663,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","header":"x-maas-subscription"} {"level":"info","ts":1789179739.663189,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179739.6632202,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.6638522,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.663873,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.6638927,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","EoS":false} {"level":"info","ts":1789179739.663906,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.6639278,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d","EoS":true} {"level":"info","ts":1789179739.6639376,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"68f125a783c1b54eb01865a1e99c7a76","span_id":"9451e6147fd0f1cc","x-request-id":"38b4d01b-a117-41df-a561-376ec60c777d"} {"level":"info","ts":1789179739.7767186,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179739.776793,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd"} {"level":"info","ts":1789179739.7773488,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd"} {"level":"info","ts":1789179739.7773654,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd"} {"level":"info","ts":1789179739.77738,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd","EoS":false} {"level":"info","ts":1789179739.777396,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd"} {"level":"info","ts":1789179739.777427,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd","EoS":true} {"level":"info","ts":1789179739.7774363,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bef4179831503151e249cb8282970c7b","span_id":"ff517e72b79928b9","x-request-id":"c1997530-2b07-48c1-858e-9508c2039edd"} {"level":"info","ts":1789179740.5493348,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179740.5494437,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179740.549459,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179740.549478,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","EoS":true} {"level":"info","ts":1789179740.5495088,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","header":"x-maas-group"} {"level":"info","ts":1789179740.549522,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","header":"x-maas-username"} {"level":"info","ts":1789179740.5495293,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","header":"x-maas-subscription"} {"level":"info","ts":1789179740.5495481,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179740.549579,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179740.5505314,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179740.5505493,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179740.5505636,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","EoS":false} {"level":"info","ts":1789179740.5505748,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179740.550599,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5","EoS":true} {"level":"info","ts":1789179740.5506084,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b104d2343ec4aeb0cb95ee57003ad94e","span_id":"f45f31a6fe5e7d05","x-request-id":"9ced1e07-31d0-40a9-be45-38cb20b6eca5"} {"level":"info","ts":1789179741.5173962,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179741.5174856,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179741.517498,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179741.5175102,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","EoS":true} {"level":"info","ts":1789179741.517529,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","header":"x-maas-group"} {"level":"info","ts":1789179741.5175395,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","header":"x-maas-username"} {"level":"info","ts":1789179741.5175445,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","header":"x-maas-subscription"} {"level":"info","ts":1789179741.5175588,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179741.5175822,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179741.5183792,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179741.5183969,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179741.518414,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","EoS":false} {"level":"info","ts":1789179741.5184312,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179741.5184557,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009","EoS":true} {"level":"info","ts":1789179741.5184658,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"70e6b3e2792c1f70e51034721e9072ad","span_id":"5e5237292d6fe26f","x-request-id":"a0f7d50e-35b9-4ab5-ac97-d57a5abd5009"} {"level":"info","ts":1789179742.2755606,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179742.2756474,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179742.2756593,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179742.2756708,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","EoS":true} {"level":"info","ts":1789179742.2756906,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","header":"x-maas-subscription"} {"level":"info","ts":1789179742.275701,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","header":"x-maas-group"} {"level":"info","ts":1789179742.2757058,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","header":"x-maas-username"} {"level":"info","ts":1789179742.275719,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179742.2757409,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179742.2765534,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179742.2765718,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179742.276584,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","EoS":false} {"level":"info","ts":1789179742.276595,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179742.2766187,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a","EoS":true} {"level":"info","ts":1789179742.2766259,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fd9d441eb3162a96c85404f4caba4f29","span_id":"0468dc189f8cf196","x-request-id":"7cad12ef-5b13-431c-a83b-a04480e19a8a"} {"level":"info","ts":1789179743.373893,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179743.37399,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"info","ts":1789179743.374005,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"info","ts":1789179743.374017,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","EoS":true} {"level":"info","ts":1789179743.3740623,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","header":"x-maas-username"} {"level":"info","ts":1789179743.3740733,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","header":"x-maas-group"} {"level":"info","ts":1789179743.3740792,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","header":"x-maas-subscription"} {"level":"info","ts":1789179743.3740923,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179743.374115,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"info","ts":1789179743.374932,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"info","ts":1789179743.37495,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"info","ts":1789179743.3749666,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","EoS":false} {"level":"info","ts":1789179743.3749788,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"info","ts":1789179743.3750029,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf","EoS":true} {"level":"info","ts":1789179743.3750162,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a42937505ab6dfa21b1e19edba752222","span_id":"eaeef5ecf5b955ed","x-request-id":"0d309196-af1b-4e9e-b5f0-c44a16379abf"} {"level":"error","ts":1789179752.3999727,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179763.1411994,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179763.1412857,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"info","ts":1789179763.141297,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"info","ts":1789179763.1413076,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","EoS":true} {"level":"info","ts":1789179763.1413276,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","header":"x-maas-subscription"} {"level":"info","ts":1789179763.1413388,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","header":"x-maas-username"} {"level":"info","ts":1789179763.1413443,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","header":"x-maas-group"} {"level":"info","ts":1789179763.1413574,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179763.141381,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"info","ts":1789179763.1422467,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"info","ts":1789179763.1422722,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"info","ts":1789179763.142297,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","EoS":false} {"level":"info","ts":1789179763.1423168,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"info","ts":1789179763.1423476,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637","EoS":true} {"level":"info","ts":1789179763.142357,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"066b753cd785ed2bddb3598f1da21977","span_id":"3220ef7860f3ebe7","x-request-id":"bfc03c3e-7e2f-423b-97db-79746982e637"} {"level":"error","ts":1789179767.401027,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789179782.4026785,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179856.8912191,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179856.8913267,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"info","ts":1789179856.8913436,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"info","ts":1789179856.8913577,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","EoS":true} {"level":"info","ts":1789179856.8913815,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","header":"x-maas-username"} {"level":"info","ts":1789179856.8913913,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","header":"x-maas-group"} {"level":"info","ts":1789179856.8913965,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","header":"x-maas-subscription"} {"level":"info","ts":1789179856.8914113,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789179856.891436,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"info","ts":1789179856.904914,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"info","ts":1789179856.9049394,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"info","ts":1789179856.904961,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","EoS":false} {"level":"info","ts":1789179856.9049757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"info","ts":1789179856.905013,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9","EoS":true} {"level":"info","ts":1789179856.9050245,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"edfccfe27e9b80dd9ed85df49ff3e82f","span_id":"62d16285000972ff","x-request-id":"bc76dafe-3f3c-4503-8e8e-4fbcb595c9d9"} {"level":"error","ts":1789179867.4103491,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179974.6664283,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179974.6665173,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"info","ts":1789179974.6665287,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"info","ts":1789179974.6665401,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","EoS":true} {"level":"info","ts":1789179974.6665597,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","header":"x-maas-group"} {"level":"info","ts":1789179974.66657,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","header":"x-maas-username"} {"level":"info","ts":1789179974.6665745,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","header":"x-maas-subscription"} {"level":"info","ts":1789179974.666588,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789179974.6666124,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"info","ts":1789179974.6673114,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"info","ts":1789179974.6673331,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"info","ts":1789179974.6673517,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","EoS":false} {"level":"info","ts":1789179974.6673667,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"info","ts":1789179974.667404,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b","EoS":true} {"level":"info","ts":1789179974.6674151,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"41137b9f5b3f311ec4a1a956c223d684","span_id":"b0e334a928ef02cb","x-request-id":"a9c55c11-03b6-42b1-a4f1-36f9e6c7d68b"} {"level":"error","ts":1789179987.426112,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789179988.631154,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179988.6312659,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.6312811,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.6313004,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","EoS":true} {"level":"info","ts":1789179988.63133,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","header":"x-maas-subscription"} {"level":"info","ts":1789179988.631342,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","header":"x-maas-group"} {"level":"info","ts":1789179988.6313498,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","header":"x-maas-username"} {"level":"info","ts":1789179988.6313708,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789179988.631402,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.6442232,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.644247,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.6442623,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","EoS":false} {"level":"info","ts":1789179988.6442735,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.6443005,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","EoS":false} {"level":"info","ts":1789179988.6443107,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.6443212,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db","EoS":true} {"level":"info","ts":1789179988.64433,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e74e3cb96816a883e9c12215e914382","span_id":"c504dcbb600234cc","x-request-id":"73d218a2-4064-45b5-976e-cd81f86382db"} {"level":"info","ts":1789179988.7806787,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179988.780751,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7807634,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7807736,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","EoS":true} {"level":"info","ts":1789179988.7807932,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","header":"x-maas-subscription"} {"level":"info","ts":1789179988.780801,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","header":"x-maas-username"} {"level":"info","ts":1789179988.780806,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","header":"x-maas-group"} {"level":"info","ts":1789179988.780819,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789179988.7808409,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7816281,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7816453,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7816622,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","EoS":false} {"level":"info","ts":1789179988.781678,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7817097,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","EoS":false} {"level":"info","ts":1789179988.7817214,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.7817326,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4","EoS":true} {"level":"info","ts":1789179988.7817392,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"af00c2adbcb64e39bd102d697e53a4cc","span_id":"eb02885f47e32eca","x-request-id":"51ebf686-8e87-4b65-9a12-3f5d06d2b3f4"} {"level":"info","ts":1789179988.9234366,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789179988.9235172,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.923528,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.9235392,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","EoS":true} {"level":"info","ts":1789179988.9235556,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","header":"x-maas-subscription"} {"level":"info","ts":1789179988.9235659,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","header":"x-maas-username"} {"level":"info","ts":1789179988.9235718,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","header":"x-maas-group"} {"level":"info","ts":1789179988.9235835,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789179988.9236066,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.9244711,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.9244876,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.924512,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","EoS":false} {"level":"info","ts":1789179988.924525,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.9245496,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","EoS":false} {"level":"info","ts":1789179988.9245648,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789179988.9245756,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf","EoS":true} {"level":"info","ts":1789179988.924587,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b398860a79bd775366bb05b233f46cb8","span_id":"74838ea262fa7898","x-request-id":"465eca96-9aa0-4c2d-b5f8-e3e9bb0c5aaf"} {"level":"info","ts":1789180001.8276517,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789180001.8277504,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8277726,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8277917,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","EoS":true} {"level":"info","ts":1789180001.8278189,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","header":"x-maas-subscription"} {"level":"info","ts":1789180001.827835,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","header":"x-maas-group"} {"level":"info","ts":1789180001.8278427,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","header":"x-maas-username"} {"level":"info","ts":1789180001.8278587,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789180001.8278825,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8308187,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.830844,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8308651,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","EoS":false} {"level":"info","ts":1789180001.8308833,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8309107,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","EoS":false} {"level":"info","ts":1789180001.830921,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8309379,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","EoS":false} {"level":"info","ts":1789180001.8309531,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"info","ts":1789180001.8309667,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9","EoS":true} {"level":"info","ts":1789180001.8309755,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f28a84cb4c22d776a853c8e2ff0d6e45","span_id":"267cc2724c130a54","x-request-id":"b5de2d6e-d9d0-4917-ae82-87a6e02b7ee9"} {"level":"error","ts":1789180002.426705,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789180012.4278467,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}