{"level":"info","ts":1789168719.6730566,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ba99b482-6030-4026-b45f-b1ce87930d78","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.6841605,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ba797bf4-6846-485d-815a-76e5d2e36690","provider":"openai","model":"e2e-legacy-migration-7933a676","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789168719.6846082,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"4bb147fb-f244-4d4d-8b20-fc990bd0a6ef","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.684636,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"4bb147fb-f244-4d4d-8b20-fc990bd0a6ef","modelName":"e2e-legacy-migration-7933a676","resolvedRefs":1} {"level":"error","ts":1789168719.684717,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a248df3d-04a5-478c-a50b-ba3dd332c141","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.6847494,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a248df3d-04a5-478c-a50b-ba3dd332c141","provider":"e2e-legacy-migration-7933a676","phase":"Pending"} {"level":"error","ts":1789168719.690494,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a248df3d-04a5-478c-a50b-ba3dd332c141","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789168719.6905296,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a248df3d-04a5-478c-a50b-ba3dd332c141","error":"ExternalModel \"e2e-legacy-migration-7933a676\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-7933a676\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.6905868,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a5239cf0-0419-4ea9-a8ca-b23c458a19c1"} {"level":"info","ts":1789168719.6906183,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a5239cf0-0419-4ea9-a8ca-b23c458a19c1","provider":"e2e-legacy-migration-7933a676","phase":"Pending"} {"level":"info","ts":1789168719.6954083,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a894cbcd-266b-4f45-96df-f103b19b77ca","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.6954417,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a894cbcd-266b-4f45-96df-f103b19b77ca","modelName":"e2e-legacy-migration-7933a676","resolvedRefs":1} {"level":"error","ts":1789168719.6983652,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ba99b482-6030-4026-b45f-b1ce87930d78","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789168719.6984112,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ba99b482-6030-4026-b45f-b1ce87930d78","error":"failed to apply Service: services \"e2e-legacy-migration-7933a676\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.6984622,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"87e253e5-4078-4c6a-ad8a-f645b021d567"} {"level":"info","ts":1789168719.6985035,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"87e253e5-4078-4c6a-ad8a-f645b021d567","name":"e2e-legacy-migration-7933a676"} {"level":"error","ts":1789168719.7024963,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a5239cf0-0419-4ea9-a8ca-b23c458a19c1","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789168719.702537,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"a5239cf0-0419-4ea9-a8ca-b23c458a19c1","error":"ExternalModel \"e2e-legacy-migration-7933a676\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-7933a676\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.7025857,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"20dfdffa-c389-4570-b9c3-ad7425b18945"} {"level":"info","ts":1789168719.702611,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"20dfdffa-c389-4570-b9c3-ad7425b18945","provider":"e2e-legacy-migration-7933a676","phase":"Pending"} {"level":"error","ts":1789168719.712696,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"20dfdffa-c389-4570-b9c3-ad7425b18945","error":"ExternalModel \"e2e-legacy-migration-7933a676\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-7933a676\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.7148206,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"0d40ceb3-4815-430f-bad0-8bf0e123dcb6","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.7148514,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"0d40ceb3-4815-430f-bad0-8bf0e123dcb6","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789168719.7149694,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"cd7df76c-6ac8-466a-81ce-921cb27097cc"} {"level":"info","ts":1789168719.7150078,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"cd7df76c-6ac8-466a-81ce-921cb27097cc","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.7152622,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"797d79c2-8303-4852-ab8d-ee045898ef8d","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.7153094,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"797d79c2-8303-4852-ab8d-ee045898ef8d","modelName":"e2e-legacy-migration-7933a676","resolvedRefs":1} {"level":"info","ts":1789168719.719019,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"87e253e5-4078-4c6a-ad8a-f645b021d567","service":"e2e-legacy-migration-7933a676","serviceEntry":"e2e-legacy-migration-7933a676","destinationRule":"e2e-legacy-migration-7933a676"} {"level":"error","ts":1789168719.7297459,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"87e253e5-4078-4c6a-ad8a-f645b021d567","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:97\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.7298214,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"bcea79c2-45dc-4718-902a-cce401dd3d93"} {"level":"info","ts":1789168719.729863,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"bcea79c2-45dc-4718-902a-cce401dd3d93","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.7403843,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"d0c38263-8fb2-4c61-9f11-8f4b9dab4498","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.7404168,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"d0c38263-8fb2-4c61-9f11-8f4b9dab4498","modelName":"e2e-legacy-migration-7933a676","resolvedRefs":1} {"level":"info","ts":1789168719.7455995,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"bcea79c2-45dc-4718-902a-cce401dd3d93","service":"e2e-legacy-migration-7933a676","serviceEntry":"e2e-legacy-migration-7933a676","destinationRule":"e2e-legacy-migration-7933a676"} {"level":"error","ts":1789168719.746639,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"cd7df76c-6ac8-466a-81ce-921cb27097cc","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789168719.746683,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"cd7df76c-6ac8-466a-81ce-921cb27097cc","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-7933a676\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.7467315,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ce07f045-6461-4be9-af20-ba35d1a1ef61"} {"level":"info","ts":1789168719.7467809,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ce07f045-6461-4be9-af20-ba35d1a1ef61","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.7590363,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"b1ad93c1-fbe3-472a-b7f5-9ba1b0890720","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.7590947,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"b1ad93c1-fbe3-472a-b7f5-9ba1b0890720","modelName":"e2e-legacy-migration-7933a676","resolvedRefs":1} {"level":"info","ts":1789168719.7615113,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ce07f045-6461-4be9-af20-ba35d1a1ef61","httpRoute":"e2e-legacy-migration-7933a676","providers":["e2e-legacy-migration-7933a676"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789168719.766521,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"ce07f045-6461-4be9-af20-ba35d1a1ef61","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.766589,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"99fced0c-8015-4667-b38a-32f108c4a9b9"} {"level":"info","ts":1789168719.7666285,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"99fced0c-8015-4667-b38a-32f108c4a9b9","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.7766328,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"2ae3a4a5-128f-4385-b86a-8e76aaf220b3","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.7766602,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"2ae3a4a5-128f-4385-b86a-8e76aaf220b3","modelName":"e2e-legacy-migration-7933a676","resolvedRefs":1} {"level":"info","ts":1789168719.777534,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"99fced0c-8015-4667-b38a-32f108c4a9b9","httpRoute":"e2e-legacy-migration-7933a676","providers":["e2e-legacy-migration-7933a676"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789168719.7815747,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"99fced0c-8015-4667-b38a-32f108c4a9b9","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-7933a676\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789168719.781657,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"12a2d153-f2e3-42ee-af10-376c9ee19676"} {"level":"info","ts":1789168719.781704,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"12a2d153-f2e3-42ee-af10-376c9ee19676","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.7932734,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"12a2d153-f2e3-42ee-af10-376c9ee19676","httpRoute":"e2e-legacy-migration-7933a676","providers":["e2e-legacy-migration-7933a676"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789168719.8765435,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"936e50aa-7a66-4249-810f-617b67fb9b5d","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168719.880246,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"936e50aa-7a66-4249-810f-617b67fb9b5d","name":"e2e-legacy-migration-7933a676"} {"level":"info","ts":1789168719.885852,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"936e50aa-7a66-4249-810f-617b67fb9b5d","provider":"openai","model":"e2e-legacy-migration-7933a676","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789168722.5196788,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"3e7e33e1-c072-4d1c-a349-60eaa1db2479","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168722.5371697,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"d8fab57b-6f6d-4731-893b-aab9f4dee6c0","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168722.5372627,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"767e341c-eef4-4e55-b721-605f8ab9cf23"} {"level":"info","ts":1789168722.5373633,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"d8fab57b-6f6d-4731-893b-aab9f4dee6c0","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168722.5378363,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"cc2721f2-4802-42bd-ac4e-5989cfe30af2","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168722.5378718,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"cc2721f2-4802-42bd-ac4e-5989cfe30af2","name":"e2e-legacy-migration-7933a676","namespace":"llm"} {"level":"info","ts":1789168722.5380423,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-7933a676","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-7933a676","reconcileID":"4809ef03-141a-44b8-986a-b6af64af6046"} {"level":"info","ts":1789168722.926337,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168722.9264336,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168722.926446,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168722.926458,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","EoS":true} {"level":"info","ts":1789168722.9264796,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","header":"x-maas-username"} {"level":"info","ts":1789168722.9264898,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","header":"x-maas-subscription"} {"level":"info","ts":1789168722.926494,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","header":"x-maas-group"} {"level":"info","ts":1789168722.9265075,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789168722.926518,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","modelName":"e2e-external-model"} {"level":"info","ts":1789168722.926531,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789168722.926553,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789168722.92657,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","authType":"apikey"} {"level":"info","ts":1789168722.926599,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168722.9862032,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168722.9862242,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168722.9862387,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","EoS":false} {"level":"info","ts":1789168722.9862497,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168722.9862735,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6","EoS":true} {"level":"info","ts":1789168722.9862807,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"704e9ea1eb75d8cb4934f3f652a307b7","span_id":"1e931d244519314a","x-request-id":"ac798373-009c-4fd8-8f22-8816d8af2ae6"} {"level":"info","ts":1789168723.0274127,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168723.027483,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.0274954,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.0275056,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","EoS":true} {"level":"info","ts":1789168723.0275247,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","header":"x-maas-group"} {"level":"info","ts":1789168723.0275342,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","header":"x-maas-subscription"} {"level":"info","ts":1789168723.0275395,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","header":"x-maas-username"} {"level":"info","ts":1789168723.0275526,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789168723.027574,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.0651987,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.0652168,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.0652306,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","EoS":false} {"level":"info","ts":1789168723.065241,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.0652647,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b","EoS":true} {"level":"info","ts":1789168723.0652735,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c2ae7621f76afd7413d7bf96b0cef32","span_id":"98ff43e48e98fcbf","x-request-id":"148d6b14-5226-4b42-a682-cf0ba014ed5b"} {"level":"info","ts":1789168723.1060328,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168723.1061108,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.106124,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.1061354,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b","EoS":true} {"level":"info","ts":1789168723.1061509,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b","header":"x-maas-subscription"} {"level":"info","ts":1789168723.1061604,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b","header":"x-maas-group"} {"level":"info","ts":1789168723.1061656,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b","header":"x-maas-username"} {"level":"info","ts":1789168723.1061945,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.151881,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.1519008,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.1519477,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b","EoS":false} {"level":"info","ts":1789168723.151966,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.1520028,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b","EoS":true} {"level":"info","ts":1789168723.152014,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"30f0022e8e47ccbc25dc0aa765e1dc02","span_id":"c0256965a425ebbb","x-request-id":"d6e2298a-6bce-4fc6-9cab-af1573815b1b"} {"level":"info","ts":1789168723.9758062,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"f95d9bb5-11a8-4712-ba4d-0277a9670d85","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789168723.9759028,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"f95d9bb5-11a8-4712-ba4d-0277a9670d85","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789168723.9759548,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"4723f5e3-dc69-41e7-ad0f-c03e8ff200ae"} {"level":"info","ts":1789168724.13467,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"f3ef5a34-73fe-46d9-94fb-89f14f23445c","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789168724.134764,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"f3ef5a34-73fe-46d9-94fb-89f14f23445c","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789168724.1346865,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"828d6628-a718-48d9-b104-0db1515e63b3"} {"level":"info","ts":1789168724.2747808,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7cfc0798-da43-47e3-9196-3a04b00b6445","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789168724.2748308,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7cfc0798-da43-47e3-9196-3a04b00b6445","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789168727.2436779,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789168734.0773168,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168734.0774546,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.0774758,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.0774913,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","EoS":true} {"level":"info","ts":1789168734.0775142,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","header":"x-maas-username"} {"level":"info","ts":1789168734.0775259,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","header":"x-maas-group"} {"level":"info","ts":1789168734.0775309,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","header":"x-maas-subscription"} {"level":"info","ts":1789168734.0775468,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789168734.0775752,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.0787811,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.0788026,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.0788178,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","EoS":false} {"level":"info","ts":1789168734.078831,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.078859,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba","EoS":true} {"level":"info","ts":1789168734.0788682,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"30efd7da0d96f98ff5a0dd41980cac86","span_id":"d6f43652785809d5","x-request-id":"13d5cf2a-4887-449a-bba4-6c6f873534ba"} {"level":"info","ts":1789168734.1134722,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168734.1135855,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1136007,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1136162,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","EoS":true} {"level":"info","ts":1789168734.1136446,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","header":"x-maas-username"} {"level":"info","ts":1789168734.1136596,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","header":"x-maas-subscription"} {"level":"info","ts":1789168734.113666,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","header":"x-maas-group"} {"level":"info","ts":1789168734.113684,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789168734.113711,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1146998,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1147187,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1147306,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","EoS":false} {"level":"info","ts":1789168734.114742,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1147666,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030","EoS":true} {"level":"info","ts":1789168734.1147747,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e8eee4bacee04a0094053a751b6e59ce","span_id":"762ae3fc3ae816bd","x-request-id":"c24beaa3-4754-4051-bdac-2eda8a9f4030"} {"level":"info","ts":1789168734.1691136,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168734.1692004,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.169212,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.1692233,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","EoS":true} {"level":"info","ts":1789168734.169245,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","header":"x-maas-group"} {"level":"info","ts":1789168734.1692555,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","header":"x-maas-subscription"} {"level":"info","ts":1789168734.1692607,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","header":"x-maas-username"} {"level":"info","ts":1789168734.169274,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789168734.1692996,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.1702049,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.170225,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.170244,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","EoS":false} {"level":"info","ts":1789168734.1702702,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.170311,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469","EoS":true} {"level":"info","ts":1789168734.170321,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"21c3a3c49b93615f6432a7b8ce7a0e2d","span_id":"593debcd36d47506","x-request-id":"3d45d238-ccc4-46a7-91fc-e04d91f3e469"} {"level":"info","ts":1789168734.2049034,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168734.2050014,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"info","ts":1789168734.205013,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"info","ts":1789168734.2050238,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","EoS":true} {"level":"info","ts":1789168734.2050514,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","header":"x-maas-group"} {"level":"info","ts":1789168734.2050614,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","header":"x-maas-username"} {"level":"info","ts":1789168734.205067,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","header":"x-maas-subscription"} {"level":"info","ts":1789168734.205082,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789168734.2051058,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"info","ts":1789168734.2060258,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"info","ts":1789168734.2060442,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"info","ts":1789168734.20606,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","EoS":false} {"level":"info","ts":1789168734.2060792,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"info","ts":1789168734.206114,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266","EoS":true} {"level":"info","ts":1789168734.2061255,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"64d5580b0ada4536e6d65c6f6b842e83","span_id":"1eecf4c46141c5c0","x-request-id":"bed35ef5-a884-44cc-b07e-2430b3e5d266"} {"level":"error","ts":1789168737.2444086,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789168747.8548093,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168747.8549404,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a48585faab5a95b89b0333f573be14b7","span_id":"308e529fe98064f2","x-request-id":"c6f9d6ee-7fe6-4dc6-92b9-53677912d15e"} {"level":"info","ts":1789168747.8549528,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a48585faab5a95b89b0333f573be14b7","span_id":"308e529fe98064f2","x-request-id":"c6f9d6ee-7fe6-4dc6-92b9-53677912d15e"} {"level":"info","ts":1789168747.854965,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a48585faab5a95b89b0333f573be14b7","span_id":"308e529fe98064f2","x-request-id":"c6f9d6ee-7fe6-4dc6-92b9-53677912d15e","EoS":true} {"level":"info","ts":1789168747.8550158,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a48585faab5a95b89b0333f573be14b7","span_id":"308e529fe98064f2","x-request-id":"c6f9d6ee-7fe6-4dc6-92b9-53677912d15e"} {"level":"info","ts":1789168747.991133,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168747.9912357,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"03a8e1f6add0f12f7fac28a8c8ffb46c","span_id":"13ae132f8df7876d","x-request-id":"68be7a52-cedc-4213-aaf9-3d94f8502fea"} {"level":"info","ts":1789168747.9912477,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"03a8e1f6add0f12f7fac28a8c8ffb46c","span_id":"13ae132f8df7876d","x-request-id":"68be7a52-cedc-4213-aaf9-3d94f8502fea"} {"level":"info","ts":1789168747.9912586,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"03a8e1f6add0f12f7fac28a8c8ffb46c","span_id":"13ae132f8df7876d","x-request-id":"68be7a52-cedc-4213-aaf9-3d94f8502fea","EoS":true} {"level":"info","ts":1789168747.9912999,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"03a8e1f6add0f12f7fac28a8c8ffb46c","span_id":"13ae132f8df7876d","x-request-id":"68be7a52-cedc-4213-aaf9-3d94f8502fea"} {"level":"info","ts":1789168748.140609,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789168748.1407151,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9bc470cca3de247e5308bbcd78016b6f","span_id":"0867ab629000f7bf","x-request-id":"19db62de-8bf7-412c-b7a0-d619c4f7dcfc"} {"level":"info","ts":1789168748.1407273,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9bc470cca3de247e5308bbcd78016b6f","span_id":"0867ab629000f7bf","x-request-id":"19db62de-8bf7-412c-b7a0-d619c4f7dcfc"} {"level":"info","ts":1789168748.1407387,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9bc470cca3de247e5308bbcd78016b6f","span_id":"0867ab629000f7bf","x-request-id":"19db62de-8bf7-412c-b7a0-d619c4f7dcfc","EoS":true} {"level":"info","ts":1789168748.1407886,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9bc470cca3de247e5308bbcd78016b6f","span_id":"0867ab629000f7bf","x-request-id":"19db62de-8bf7-412c-b7a0-d619c4f7dcfc"} {"level":"error","ts":1789168752.2455142,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789168767.2460544,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789169825.2948847,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789169825.2950249,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.2950385,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.2951038,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","EoS":true} {"level":"info","ts":1789169825.295142,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","header":"x-maas-group"} {"level":"info","ts":1789169825.295155,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","header":"x-maas-username"} {"level":"info","ts":1789169825.2951608,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","header":"x-maas-subscription"} {"level":"info","ts":1789169825.295176,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789169825.2952032,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.2960837,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.2961032,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.2961168,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","EoS":false} {"level":"info","ts":1789169825.2961276,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.2961488,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f","EoS":true} {"level":"info","ts":1789169825.2961588,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"df7a22b4319624d96b5f427ad2546ae4","span_id":"63d3047fd3d65884","x-request-id":"aa7d4a9b-f943-4e1b-91c3-77a1615acf9f"} {"level":"info","ts":1789169825.3554962,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789169825.355607,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"info","ts":1789169825.3556197,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"info","ts":1789169825.3556316,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","EoS":true} {"level":"info","ts":1789169825.3556564,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","header":"x-maas-username"} {"level":"info","ts":1789169825.3556662,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","header":"x-maas-subscription"} {"level":"info","ts":1789169825.355672,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","header":"x-maas-group"} {"level":"info","ts":1789169825.3556867,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789169825.3557138,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"info","ts":1789169825.356641,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"info","ts":1789169825.3566642,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"info","ts":1789169825.3566854,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","EoS":false} {"level":"info","ts":1789169825.356705,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"info","ts":1789169825.356744,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad","EoS":true} {"level":"info","ts":1789169825.3567548,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7adcbe17cf2dde8be5c8efaed171540d","span_id":"b1116706764dfcf0","x-request-id":"b5d3efa2-34c9-445e-bf88-78708feee3ad"} {"level":"error","ts":1789169837.3638222,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789170177.926674,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170177.9268117,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170177.9268296,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170177.9268482,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","EoS":true} {"level":"info","ts":1789170177.9268825,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","header":"x-maas-username"} {"level":"info","ts":1789170177.9268954,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","header":"x-maas-subscription"} {"level":"info","ts":1789170177.9269042,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","header":"x-maas-group"} {"level":"info","ts":1789170177.926952,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170177.9269922,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170177.940232,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170177.9402676,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170177.940284,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","EoS":false} {"level":"info","ts":1789170177.9402978,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170177.940326,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff","EoS":true} {"level":"info","ts":1789170177.9403334,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b841d0215037c91ed4c1555d3afc2fb3","span_id":"8a282c286181b542","x-request-id":"df18841f-39bf-4ba8-be37-e473362913ff"} {"level":"info","ts":1789170178.0803926,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170178.0804985,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.080512,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.0805242,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","EoS":true} {"level":"info","ts":1789170178.0805504,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","header":"x-maas-group"} {"level":"info","ts":1789170178.080601,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","header":"x-maas-username"} {"level":"info","ts":1789170178.080614,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","header":"x-maas-subscription"} {"level":"info","ts":1789170178.0806293,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170178.080657,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.081669,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.081689,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.0817056,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","EoS":false} {"level":"info","ts":1789170178.0817177,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.081746,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528","EoS":true} {"level":"info","ts":1789170178.0817566,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"91c6c83e4cfb06a68114255fc8d8926e","span_id":"845847f52e51b00a","x-request-id":"1c0e3bb3-6032-4f03-ae57-bfe31af0c528"} {"level":"info","ts":1789170178.2215703,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170178.221671,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.2216842,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.2216952,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","EoS":true} {"level":"info","ts":1789170178.22172,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","header":"x-maas-username"} {"level":"info","ts":1789170178.2217293,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","header":"x-maas-group"} {"level":"info","ts":1789170178.2217343,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","header":"x-maas-subscription"} {"level":"info","ts":1789170178.2217503,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170178.2217739,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.2240522,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.2240698,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.224083,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","EoS":false} {"level":"info","ts":1789170178.2241008,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.224128,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15","EoS":true} {"level":"info","ts":1789170178.2241375,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f9c92dc9b3c541ff44ced31c24f0d45","span_id":"e99cacf0c9e81c2b","x-request-id":"5a6d63c0-451f-467d-9b9e-a1aca4ca9d15"} {"level":"info","ts":1789170178.3619733,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170178.3620756,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.3620875,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.3620987,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","EoS":true} {"level":"info","ts":1789170178.3621223,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","header":"x-maas-group"} {"level":"info","ts":1789170178.3621316,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","header":"x-maas-subscription"} {"level":"info","ts":1789170178.3621366,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","header":"x-maas-username"} {"level":"info","ts":1789170178.3621516,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170178.3621745,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.3631363,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.363154,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.3632028,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","EoS":false} {"level":"info","ts":1789170178.36322,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.363244,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd","EoS":true} {"level":"info","ts":1789170178.3632548,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f682e8a205693347b0d8d8b5e20d3cde","span_id":"8ec32f2b544056ef","x-request-id":"9069441c-42f3-4e6a-8231-040d29ca4fbd"} {"level":"info","ts":1789170178.498907,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170178.4990237,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"info","ts":1789170178.499036,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"info","ts":1789170178.4990473,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","EoS":true} {"level":"info","ts":1789170178.4990697,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","header":"x-maas-subscription"} {"level":"info","ts":1789170178.4990814,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","header":"x-maas-group"} {"level":"info","ts":1789170178.49909,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","header":"x-maas-username"} {"level":"info","ts":1789170178.4991078,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170178.4991372,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"info","ts":1789170178.500089,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"info","ts":1789170178.5001063,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"info","ts":1789170178.5001202,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","EoS":false} {"level":"info","ts":1789170178.5001388,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"info","ts":1789170178.500164,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec","EoS":true} {"level":"info","ts":1789170178.500173,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"083c1fd26081d0f6439a3db8edc83b08","span_id":"79ae8c54b55a28c2","x-request-id":"07f5003b-64e4-4639-9c3c-6a405e0894ec"} {"level":"error","ts":1789170192.4022586,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789170193.4798954,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170193.480025,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.4800396,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.4800518,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","EoS":true} {"level":"info","ts":1789170193.4800775,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","header":"x-maas-group"} {"level":"info","ts":1789170193.480088,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","header":"x-maas-subscription"} {"level":"info","ts":1789170193.4800928,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","header":"x-maas-username"} {"level":"info","ts":1789170193.4801087,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170193.4801397,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.4818459,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.4818661,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.4818807,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","EoS":false} {"level":"info","ts":1789170193.4818916,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.4819348,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a","EoS":true} {"level":"info","ts":1789170193.4819455,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c45d512bb43df095baa813b815848be7","span_id":"beb553bd8109a3c8","x-request-id":"53df0810-e8bc-4a87-8e43-b5b2ff80e26a"} {"level":"info","ts":1789170193.579274,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170193.5793743,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a"} {"level":"info","ts":1789170193.5803497,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a"} {"level":"info","ts":1789170193.580368,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a"} {"level":"info","ts":1789170193.5803812,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a","EoS":false} {"level":"info","ts":1789170193.5803938,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a"} {"level":"info","ts":1789170193.5804198,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a","EoS":true} {"level":"info","ts":1789170193.5804296,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a2c2f0c620899f07d285afd3aeda37e6","span_id":"03d81ae7c0d93652","x-request-id":"1a7378cd-71fc-431d-9975-64f26433b92a"} {"level":"info","ts":1789170194.3014462,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170194.3015523,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170194.3015697,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170194.3015823,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","EoS":true} {"level":"info","ts":1789170194.3016088,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","header":"x-maas-group"} {"level":"info","ts":1789170194.3016183,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","header":"x-maas-subscription"} {"level":"info","ts":1789170194.301623,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","header":"x-maas-username"} {"level":"info","ts":1789170194.301638,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170194.30167,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170194.302509,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170194.302521,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170194.3025334,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","EoS":false} {"level":"info","ts":1789170194.3025444,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170194.3025675,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0","EoS":true} {"level":"info","ts":1789170194.3025792,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"05dab7b27503e731adaa7a2078d0ba79","span_id":"9e58ee5a7a99494c","x-request-id":"c4a9f8d9-4f7c-413d-a874-c3d77c4d3fc0"} {"level":"info","ts":1789170195.205879,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170195.2060094,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.2060244,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.2060368,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","EoS":true} {"level":"info","ts":1789170195.206061,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","header":"x-maas-group"} {"level":"info","ts":1789170195.2060702,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","header":"x-maas-username"} {"level":"info","ts":1789170195.2060838,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","header":"x-maas-subscription"} {"level":"info","ts":1789170195.2061,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170195.206126,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.2070093,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.2070305,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.207048,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","EoS":false} {"level":"info","ts":1789170195.207062,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.207087,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963","EoS":true} {"level":"info","ts":1789170195.207096,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9323c3bac7df2e2b157f31d5faec15e7","span_id":"1f522f88760274e1","x-request-id":"8dd09b14-aa01-4b25-88bf-e063ff602963"} {"level":"info","ts":1789170195.885717,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170195.8858247,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170195.88584,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170195.885852,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","EoS":true} {"level":"info","ts":1789170195.885876,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","header":"x-maas-subscription"} {"level":"info","ts":1789170195.8858871,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","header":"x-maas-username"} {"level":"info","ts":1789170195.8858922,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","header":"x-maas-group"} {"level":"info","ts":1789170195.8859081,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170195.8859665,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170195.8867795,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170195.8868048,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170195.8868167,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","EoS":false} {"level":"info","ts":1789170195.8868313,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170195.8868558,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c","EoS":true} {"level":"info","ts":1789170195.886865,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fac860180b5c9f5c0c981921d17a284a","span_id":"3084d964992deefb","x-request-id":"4f4e0835-6efd-4311-89b5-9dca26beeb1c"} {"level":"info","ts":1789170196.85758,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170196.8576844,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"info","ts":1789170196.8576965,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"info","ts":1789170196.8577116,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","EoS":true} {"level":"info","ts":1789170196.857742,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","header":"x-maas-group"} {"level":"info","ts":1789170196.8577588,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","header":"x-maas-subscription"} {"level":"info","ts":1789170196.8577647,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","header":"x-maas-username"} {"level":"info","ts":1789170196.857781,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170196.8578074,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"info","ts":1789170196.8586466,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"info","ts":1789170196.8586648,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"info","ts":1789170196.858679,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","EoS":false} {"level":"info","ts":1789170196.8586905,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"info","ts":1789170196.8587217,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0","EoS":true} {"level":"info","ts":1789170196.8587313,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bb15e928fc0cf0d52751b683c4ac35c3","span_id":"f813b542a0cfa0cd","x-request-id":"c87131d2-4350-434c-ae10-babbb94270d0"} {"level":"error","ts":1789170207.402857,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789170220.4261882,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170220.4262989,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"info","ts":1789170220.4263124,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"info","ts":1789170220.4263253,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","EoS":true} {"level":"info","ts":1789170220.4263504,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","header":"x-maas-username"} {"level":"info","ts":1789170220.4263608,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","header":"x-maas-group"} {"level":"info","ts":1789170220.4263659,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","header":"x-maas-subscription"} {"level":"info","ts":1789170220.4263866,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170220.4264193,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"info","ts":1789170220.4273014,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"info","ts":1789170220.4273212,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"info","ts":1789170220.4273362,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","EoS":false} {"level":"info","ts":1789170220.4273462,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"info","ts":1789170220.427371,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8","EoS":true} {"level":"info","ts":1789170220.4273806,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4b2ff6125a009a2c205add7272744087","span_id":"2859a5ad52ce55fa","x-request-id":"ee4f9ded-955b-4156-b945-62d25759b3b8"} {"level":"error","ts":1789170232.405245,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789170312.9835446,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170312.9836464,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"info","ts":1789170312.9836593,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"info","ts":1789170312.983671,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","EoS":true} {"level":"info","ts":1789170312.9836962,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","header":"x-maas-subscription"} {"level":"info","ts":1789170312.9837065,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","header":"x-maas-group"} {"level":"info","ts":1789170312.983711,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","header":"x-maas-username"} {"level":"info","ts":1789170312.9837255,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789170312.9837499,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"info","ts":1789170312.9970174,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"info","ts":1789170312.9970422,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"info","ts":1789170312.9970605,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","EoS":false} {"level":"info","ts":1789170312.9970713,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"info","ts":1789170312.9970944,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f","EoS":true} {"level":"info","ts":1789170312.997104,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b4f23a7414429ba1eb10cf8a6a318de8","span_id":"a12da000a59b2adc","x-request-id":"cd1bf0a2-f90e-4cf5-b491-315fe01d4f1f"} {"level":"error","ts":1789170327.4161506,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789170423.9050748,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170423.905175,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"info","ts":1789170423.905188,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"info","ts":1789170423.905199,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","EoS":true} {"level":"info","ts":1789170423.9052234,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","header":"x-maas-subscription"} {"level":"info","ts":1789170423.9052331,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","header":"x-maas-group"} {"level":"info","ts":1789170423.905238,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","header":"x-maas-username"} {"level":"info","ts":1789170423.9052877,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789170423.9053118,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"info","ts":1789170423.906289,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"info","ts":1789170423.90631,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"info","ts":1789170423.906331,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","EoS":false} {"level":"info","ts":1789170423.9063528,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"info","ts":1789170423.90639,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134","EoS":true} {"level":"info","ts":1789170423.9064033,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"26fdaf4707a8fc13b4e26223831e8668","span_id":"631b2ad61bb83ea6","x-request-id":"166812d7-de6f-469d-9be0-6f4bab231134"} {"level":"error","ts":1789170437.4281273,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789170437.4561303,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170437.4562278,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.4562402,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.4562516,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","EoS":true} {"level":"info","ts":1789170437.456275,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","header":"x-maas-subscription"} {"level":"info","ts":1789170437.456289,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","header":"x-maas-group"} {"level":"info","ts":1789170437.456296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","header":"x-maas-username"} {"level":"info","ts":1789170437.4563162,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789170437.4563515,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.4704978,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.4705236,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.4705415,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","EoS":false} {"level":"info","ts":1789170437.4705613,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.4705992,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","EoS":false} {"level":"info","ts":1789170437.4706094,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.470659,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623","EoS":true} {"level":"info","ts":1789170437.4706707,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8067e809de9c02eda8203fcf95f178e6","span_id":"146c422c70fc5f3d","x-request-id":"d3fe7d27-fac1-4a2a-817c-d127fd788623"} {"level":"info","ts":1789170437.6079242,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170437.6080382,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.608054,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.608072,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","EoS":true} {"level":"info","ts":1789170437.608107,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","header":"x-maas-group"} {"level":"info","ts":1789170437.6081214,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","header":"x-maas-username"} {"level":"info","ts":1789170437.6081297,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","header":"x-maas-subscription"} {"level":"info","ts":1789170437.6081498,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789170437.6081889,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.6093397,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.6093645,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.6093814,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","EoS":false} {"level":"info","ts":1789170437.609395,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.6094232,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044","EoS":true} {"level":"info","ts":1789170437.6094356,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"68a4a9507025bb10da183a686def9afa","span_id":"909ea6943eb4bd9b","x-request-id":"b72082d5-1afc-4f82-9bd1-615414d08044"} {"level":"info","ts":1789170437.747619,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170437.7477145,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.7477279,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.7477386,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","EoS":true} {"level":"info","ts":1789170437.7477624,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","header":"x-maas-group"} {"level":"info","ts":1789170437.7477725,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","header":"x-maas-subscription"} {"level":"info","ts":1789170437.7477777,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","header":"x-maas-username"} {"level":"info","ts":1789170437.7477927,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789170437.7478187,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.7506506,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.750675,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.750696,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","EoS":false} {"level":"info","ts":1789170437.7507122,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.7507513,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","EoS":false} {"level":"info","ts":1789170437.7507634,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170437.7507842,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76","EoS":true} {"level":"info","ts":1789170437.7507975,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e45a94394d5fca12551f8ae0fbce4e9","span_id":"093c63c707d09333","x-request-id":"6d054e78-4b40-414c-95d7-25498c31df76"} {"level":"info","ts":1789170450.5468898,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789170450.547024,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"info","ts":1789170450.5470371,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"info","ts":1789170450.5470512,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","EoS":true} {"level":"info","ts":1789170450.5470734,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","header":"x-maas-subscription"} {"level":"info","ts":1789170450.5470827,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","header":"x-maas-username"} {"level":"info","ts":1789170450.5470874,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","header":"x-maas-group"} {"level":"info","ts":1789170450.5471044,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789170450.5471368,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"info","ts":1789170450.5490248,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"info","ts":1789170450.5490499,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"info","ts":1789170450.5490723,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","EoS":false} {"level":"info","ts":1789170450.5490916,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"info","ts":1789170450.5491273,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc","EoS":true} {"level":"info","ts":1789170450.549141,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6b64db1b25cc3d16a5670cc7fe11dfe3","span_id":"490feac9fdf814bb","x-request-id":"c8c90aaf-dd6a-4f72-91d2-4182c46fa8fc"} {"level":"error","ts":1789170452.4301374,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}