{"level":"error","ts":1789162180.9844463,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"ea4df476-4c09-49d1-abad-c7af34eb1f32","error":"ExternalModel \"e2e-legacy-migration-f897f66e\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-f897f66e\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162180.9845026,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2c3ebb99-d5a6-47b0-8717-9669d67c2ac3"} {"level":"info","ts":1789162180.984526,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2c3ebb99-d5a6-47b0-8717-9669d67c2ac3","provider":"e2e-legacy-migration-f897f66e","phase":"Pending"} {"level":"info","ts":1789162180.9941652,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"cdc097c9-73a5-4163-a1b4-ccb7117912b9","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162180.9943254,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"399c83d6-ca2a-43a8-b8c6-1e9386a912a7","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162180.9943542,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"399c83d6-ca2a-43a8-b8c6-1e9386a912a7","modelName":"e2e-legacy-migration-f897f66e","resolvedRefs":1} {"level":"info","ts":1789162181.0014606,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"cdc097c9-73a5-4163-a1b4-ccb7117912b9","name":"e2e-legacy-migration-f897f66e"} {"level":"error","ts":1789162181.001448,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"11266919-5037-4316-bc2e-c44c4a77eb07","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-f897f66e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789162181.0015774,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"11266919-5037-4316-bc2e-c44c4a77eb07","error":"failed to apply Service: services \"e2e-legacy-migration-f897f66e\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.001637,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5f51145b-3ab2-4e9e-8f35-8875db509473"} {"level":"info","ts":1789162181.0017147,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5f51145b-3ab2-4e9e-8f35-8875db509473","name":"e2e-legacy-migration-f897f66e"} {"level":"error","ts":1789162181.0216708,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2c3ebb99-d5a6-47b0-8717-9669d67c2ac3","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-f897f66e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789162181.0217125,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2c3ebb99-d5a6-47b0-8717-9669d67c2ac3","error":"ExternalModel \"e2e-legacy-migration-f897f66e\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-f897f66e\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.0217748,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"295a0f35-24d5-4e50-8681-3e481db26cb1"} {"level":"info","ts":1789162181.0218062,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"295a0f35-24d5-4e50-8681-3e481db26cb1","provider":"e2e-legacy-migration-f897f66e","phase":"Pending"} {"level":"info","ts":1789162181.0254636,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5f51145b-3ab2-4e9e-8f35-8875db509473","service":"e2e-legacy-migration-f897f66e","serviceEntry":"e2e-legacy-migration-f897f66e","destinationRule":"e2e-legacy-migration-f897f66e"} {"level":"info","ts":1789162181.0271106,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"08a0af52-b04a-4a15-bd16-84d2912535e1","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162181.0271423,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"08a0af52-b04a-4a15-bd16-84d2912535e1","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789162181.0271533,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"6e6354a5-6221-40b2-b86b-294ce5ea4f54","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162181.0271797,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"6e6354a5-6221-40b2-b86b-294ce5ea4f54","modelName":"e2e-legacy-migration-f897f66e","resolvedRefs":1} {"level":"info","ts":1789162181.0291727,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"cdc097c9-73a5-4163-a1b4-ccb7117912b9","provider":"openai","model":"e2e-legacy-migration-f897f66e","targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789162181.0373132,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"295a0f35-24d5-4e50-8681-3e481db26cb1","error":"ExternalModel \"e2e-legacy-migration-f897f66e\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-f897f66e\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.0373642,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"bb38ea2a-6ca3-4361-9f66-4fdd2b6c9de7"} {"level":"info","ts":1789162181.0373962,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"bb38ea2a-6ca3-4361-9f66-4fdd2b6c9de7","name":"e2e-legacy-migration-f897f66e"} {"level":"error","ts":1789162181.0496275,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5f51145b-3ab2-4e9e-8f35-8875db509473","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-f897f66e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:97\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.049698,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2be9501a-9644-4289-a8e2-76832a1e2f82"} {"level":"info","ts":1789162181.049735,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2be9501a-9644-4289-a8e2-76832a1e2f82","name":"e2e-legacy-migration-f897f66e"} {"level":"info","ts":1789162181.088766,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"abc7489d-3492-4fdf-8cf0-f73c0d6eac19","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162181.088796,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"abc7489d-3492-4fdf-8cf0-f73c0d6eac19","modelName":"e2e-legacy-migration-f897f66e","resolvedRefs":1} {"level":"info","ts":1789162181.090719,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"2be9501a-9644-4289-a8e2-76832a1e2f82","service":"e2e-legacy-migration-f897f66e","serviceEntry":"e2e-legacy-migration-f897f66e","destinationRule":"e2e-legacy-migration-f897f66e"} {"level":"error","ts":1789162181.0995405,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"bb38ea2a-6ca3-4361-9f66-4fdd2b6c9de7","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-f897f66e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789162181.099584,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"bb38ea2a-6ca3-4361-9f66-4fdd2b6c9de7","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-f897f66e\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.0996346,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"dc6b5c93-be7b-450d-9ae7-4ff159212474"} {"level":"info","ts":1789162181.0996916,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"dc6b5c93-be7b-450d-9ae7-4ff159212474","name":"e2e-legacy-migration-f897f66e"} {"level":"info","ts":1789162181.1149633,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"47d1b08f-0e8a-4034-b53d-6f4c95ee5510","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162181.1150012,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"47d1b08f-0e8a-4034-b53d-6f4c95ee5510","modelName":"e2e-legacy-migration-f897f66e","resolvedRefs":1} {"level":"info","ts":1789162181.1235483,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"dc6b5c93-be7b-450d-9ae7-4ff159212474","httpRoute":"e2e-legacy-migration-f897f66e","providers":["e2e-legacy-migration-f897f66e"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789162181.1414325,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"dc6b5c93-be7b-450d-9ae7-4ff159212474","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-f897f66e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.1415606,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"d3bcaa5d-ef8b-47e1-b6c6-0a7f14a79fd1"} {"level":"info","ts":1789162181.1416352,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"d3bcaa5d-ef8b-47e1-b6c6-0a7f14a79fd1","name":"e2e-legacy-migration-f897f66e"} {"level":"info","ts":1789162181.1569698,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"365cf4c7-0211-46f0-8087-f6a80cf8a37a","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162181.1570125,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"365cf4c7-0211-46f0-8087-f6a80cf8a37a","modelName":"e2e-legacy-migration-f897f66e","resolvedRefs":1} {"level":"info","ts":1789162181.1633625,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"d3bcaa5d-ef8b-47e1-b6c6-0a7f14a79fd1","httpRoute":"e2e-legacy-migration-f897f66e","providers":["e2e-legacy-migration-f897f66e"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789162181.1755192,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"d3bcaa5d-ef8b-47e1-b6c6-0a7f14a79fd1","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-f897f66e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789162181.175616,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"69f07720-6802-40c3-847c-e725bade8b8c"} {"level":"info","ts":1789162181.1756759,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"69f07720-6802-40c3-847c-e725bade8b8c","name":"e2e-legacy-migration-f897f66e"} {"level":"info","ts":1789162181.1890452,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"69f07720-6802-40c3-847c-e725bade8b8c","httpRoute":"e2e-legacy-migration-f897f66e","providers":["e2e-legacy-migration-f897f66e"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789162181.320803,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5a825155-34ef-413b-8587-a635f36f8d87","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162181.326793,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5a825155-34ef-413b-8587-a635f36f8d87","name":"e2e-legacy-migration-f897f66e"} {"level":"info","ts":1789162181.3492785,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5a825155-34ef-413b-8587-a635f36f8d87","provider":"openai","model":"e2e-legacy-migration-f897f66e","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789162183.9100459,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"b810e824-a5ae-42b5-be74-18ac19365c6a","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162183.9180574,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"0d34cbde-4fcf-437b-8d79-7d801d0cc3b6","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162183.9181082,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"0d34cbde-4fcf-437b-8d79-7d801d0cc3b6","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162183.9181128,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"ea1a9137-46a5-4286-8af2-5040814ccc5a"} {"level":"info","ts":1789162183.9191139,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5133d35d-b04a-4164-87f1-d9bca6abd247","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162183.919143,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"5133d35d-b04a-4164-87f1-d9bca6abd247","name":"e2e-legacy-migration-f897f66e","namespace":"llm"} {"level":"info","ts":1789162183.9192212,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-f897f66e","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-f897f66e","reconcileID":"f8e6e14b-a170-4753-8e3f-6b918d0211da"} {"level":"info","ts":1789162184.3480835,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162184.348202,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.3482187,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.3482327,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","EoS":true} {"level":"info","ts":1789162184.348262,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","header":"x-maas-subscription"} {"level":"info","ts":1789162184.3482723,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","header":"x-maas-username"} {"level":"info","ts":1789162184.3482826,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","header":"x-maas-group"} {"level":"info","ts":1789162184.3482966,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789162184.348308,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","modelName":"e2e-external-model"} {"level":"info","ts":1789162184.3483188,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789162184.3484368,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789162184.3484688,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","authType":"apikey"} {"level":"info","ts":1789162184.3485143,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.406161,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.406192,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.4062116,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","EoS":false} {"level":"info","ts":1789162184.4062257,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.406259,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d","EoS":true} {"level":"info","ts":1789162184.4062676,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"810320c8dce81c8ccb7b79c51b3d4af8","span_id":"dd27211eb10c543f","x-request-id":"59a577c5-cd4e-405e-aece-de80d59eb60d"} {"level":"info","ts":1789162184.4498048,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162184.449982,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.4500022,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.450023,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","EoS":true} {"level":"info","ts":1789162184.4500535,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","header":"x-maas-group"} {"level":"info","ts":1789162184.4500635,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","header":"x-maas-username"} {"level":"info","ts":1789162184.450071,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","header":"x-maas-subscription"} {"level":"info","ts":1789162184.4500902,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789162184.4501255,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.4926305,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.4926553,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.4926684,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","EoS":false} {"level":"info","ts":1789162184.4926865,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.492717,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369","EoS":true} {"level":"info","ts":1789162184.4927244,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"585f4e1664511f3bb11bcbff9cfa9da4","span_id":"f21b937395a907fa","x-request-id":"d1707c21-30d6-4dfe-8a36-5588d7f33369"} {"level":"info","ts":1789162184.5326734,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162184.5327911,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162184.5328083,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162184.5328522,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80","EoS":true} {"level":"info","ts":1789162184.5328827,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80","header":"x-maas-group"} {"level":"info","ts":1789162184.532897,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80","header":"x-maas-subscription"} {"level":"info","ts":1789162184.5329063,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80","header":"x-maas-username"} {"level":"info","ts":1789162184.5329437,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162184.5691001,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162184.5691323,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162184.5691545,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80","EoS":false} {"level":"info","ts":1789162184.5691724,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162184.5692134,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80","EoS":true} {"level":"info","ts":1789162184.5692277,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ea206776e225fe77a305d1e0d39dd0b6","span_id":"b0ac71c036898819","x-request-id":"387633bd-9ca1-4b2f-852b-1309df328b80"} {"level":"info","ts":1789162185.500375,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"27adcdb9-d7dc-4c54-b2f5-a787951a0454","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789162185.5004265,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"27adcdb9-d7dc-4c54-b2f5-a787951a0454","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789162185.50049,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"9cb565dc-654b-4a0b-885f-5768e6d2238a"} {"level":"info","ts":1789162185.6855192,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"d6ce3462-b4cb-474f-bfaa-32f1931e662b","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789162185.6855578,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"d6ce3462-b4cb-474f-bfaa-32f1931e662b","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789162185.6856275,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"cf24428a-6357-427f-8fad-6771a2ed749b"} {"level":"info","ts":1789162185.8417253,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"f6f39c2c-4503-46a2-a922-ccd8f01df116","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789162185.841793,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"f6f39c2c-4503-46a2-a922-ccd8f01df116","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789162187.239441,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789162202.240926,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789162207.024637,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162207.0247774,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.024793,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.024807,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","EoS":true} {"level":"info","ts":1789162207.0248635,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","header":"x-maas-group"} {"level":"info","ts":1789162207.0248778,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","header":"x-maas-subscription"} {"level":"info","ts":1789162207.024883,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","header":"x-maas-username"} {"level":"info","ts":1789162207.0248976,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789162207.0249233,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.026652,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.0266752,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.026697,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","EoS":false} {"level":"info","ts":1789162207.0267127,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.026746,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb","EoS":true} {"level":"info","ts":1789162207.0267577,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3df3ddafba3c53c81a9d096dbf0f2f3b","span_id":"11203b7ab87e92d7","x-request-id":"0bbd8828-1974-4cb1-aa19-0c33fd3163eb"} {"level":"info","ts":1789162207.1081147,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162207.10822,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.1082323,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.1082437,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","EoS":true} {"level":"info","ts":1789162207.1082685,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","header":"x-maas-subscription"} {"level":"info","ts":1789162207.1082785,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","header":"x-maas-group"} {"level":"info","ts":1789162207.1082833,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","header":"x-maas-username"} {"level":"info","ts":1789162207.108298,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789162207.1083243,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.109111,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.1091347,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.1091502,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","EoS":false} {"level":"info","ts":1789162207.1091645,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.1091971,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109","EoS":true} {"level":"info","ts":1789162207.1092086,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"530305a8953f56999ccc997958e943a3","span_id":"7ac30ccbd1671c90","x-request-id":"a70fa687-3418-447a-b739-e33747171109"} {"level":"info","ts":1789162207.1787982,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162207.1789198,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.1789372,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.1789513,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","EoS":true} {"level":"info","ts":1789162207.1789784,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","header":"x-maas-subscription"} {"level":"info","ts":1789162207.1789882,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","header":"x-maas-username"} {"level":"info","ts":1789162207.178993,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","header":"x-maas-group"} {"level":"info","ts":1789162207.1790102,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789162207.1790407,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.1797707,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.1797936,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.1798122,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","EoS":false} {"level":"info","ts":1789162207.1798623,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.179897,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac","EoS":true} {"level":"info","ts":1789162207.1799085,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"193eddf2fcac18c66ba0a5b05ddfed3a","span_id":"a9bd48327d3c740e","x-request-id":"73aaa144-3905-4624-a79c-798c783ef4ac"} {"level":"info","ts":1789162207.2216501,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162207.2217684,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"info","ts":1789162207.221786,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"info","ts":1789162207.2218013,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","EoS":true} {"level":"info","ts":1789162207.2218423,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","header":"x-maas-group"} {"level":"info","ts":1789162207.2218537,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","header":"x-maas-subscription"} {"level":"info","ts":1789162207.2218685,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","header":"x-maas-username"} {"level":"info","ts":1789162207.2218852,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789162207.2219136,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"info","ts":1789162207.2225873,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"info","ts":1789162207.222607,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"info","ts":1789162207.222622,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","EoS":false} {"level":"info","ts":1789162207.222633,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"info","ts":1789162207.2226598,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c","EoS":true} {"level":"info","ts":1789162207.2226734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6c603e0854cb08dfabbda546ba691a70","span_id":"d3ccdfcec398a37d","x-request-id":"242382e0-bac6-4760-a35f-56912a90834c"} {"level":"error","ts":1789162217.2417614,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789162222.3094878,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162222.3096097,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b7282d26c2e50d7e46c72c82e7e738cc","span_id":"a0f09137f272ea1a","x-request-id":"b5b60071-42ee-40e3-bfb4-e7dba920593c"} {"level":"info","ts":1789162222.3096356,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b7282d26c2e50d7e46c72c82e7e738cc","span_id":"a0f09137f272ea1a","x-request-id":"b5b60071-42ee-40e3-bfb4-e7dba920593c"} {"level":"info","ts":1789162222.3096523,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b7282d26c2e50d7e46c72c82e7e738cc","span_id":"a0f09137f272ea1a","x-request-id":"b5b60071-42ee-40e3-bfb4-e7dba920593c","EoS":true} {"level":"info","ts":1789162222.309702,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b7282d26c2e50d7e46c72c82e7e738cc","span_id":"a0f09137f272ea1a","x-request-id":"b5b60071-42ee-40e3-bfb4-e7dba920593c"} {"level":"info","ts":1789162222.4681346,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162222.4682553,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"cea6ff9a946f6139c83397717167dcaa","span_id":"afa366a811b27220","x-request-id":"63d9ca70-592b-4315-ac9d-a0026d918ea9"} {"level":"info","ts":1789162222.4682755,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"cea6ff9a946f6139c83397717167dcaa","span_id":"afa366a811b27220","x-request-id":"63d9ca70-592b-4315-ac9d-a0026d918ea9"} {"level":"info","ts":1789162222.4682899,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"cea6ff9a946f6139c83397717167dcaa","span_id":"afa366a811b27220","x-request-id":"63d9ca70-592b-4315-ac9d-a0026d918ea9","EoS":true} {"level":"info","ts":1789162222.468337,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"cea6ff9a946f6139c83397717167dcaa","span_id":"afa366a811b27220","x-request-id":"63d9ca70-592b-4315-ac9d-a0026d918ea9"} {"level":"info","ts":1789162222.628358,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789162222.6284719,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1fc1dbba6ecd9815014645211bbdcfeb","span_id":"fbee8ea4fc2265e3","x-request-id":"3a879144-3dd2-4485-8fe2-138d7ea2aec0"} {"level":"info","ts":1789162222.6284857,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1fc1dbba6ecd9815014645211bbdcfeb","span_id":"fbee8ea4fc2265e3","x-request-id":"3a879144-3dd2-4485-8fe2-138d7ea2aec0"} {"level":"info","ts":1789162222.628498,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1fc1dbba6ecd9815014645211bbdcfeb","span_id":"fbee8ea4fc2265e3","x-request-id":"3a879144-3dd2-4485-8fe2-138d7ea2aec0","EoS":true} {"level":"info","ts":1789162222.6285477,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1fc1dbba6ecd9815014645211bbdcfeb","span_id":"fbee8ea4fc2265e3","x-request-id":"3a879144-3dd2-4485-8fe2-138d7ea2aec0"} {"level":"error","ts":1789162237.2433925,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163197.0214355,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163197.0215416,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.0215538,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.0215652,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","EoS":true} {"level":"info","ts":1789163197.0215917,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","header":"x-maas-subscription"} {"level":"info","ts":1789163197.0216014,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","header":"x-maas-group"} {"level":"info","ts":1789163197.0216072,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","header":"x-maas-username"} {"level":"info","ts":1789163197.0216222,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789163197.0216444,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.0224571,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.022496,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.0225208,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","EoS":false} {"level":"info","ts":1789163197.0225422,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.0225718,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a","EoS":true} {"level":"info","ts":1789163197.0225804,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"622b72f97ae51f36bfee8663bd3accb6","span_id":"b2945d14804a20c7","x-request-id":"580452a5-e544-4966-b3c3-ad0594fc455a"} {"level":"info","ts":1789163197.0672972,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163197.0673964,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"info","ts":1789163197.0674098,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"info","ts":1789163197.067422,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","EoS":true} {"level":"info","ts":1789163197.0674472,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","header":"x-maas-subscription"} {"level":"info","ts":1789163197.0674555,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","header":"x-maas-username"} {"level":"info","ts":1789163197.0674603,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","header":"x-maas-group"} {"level":"info","ts":1789163197.0674744,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789163197.0675013,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"info","ts":1789163197.068179,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"info","ts":1789163197.068196,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"info","ts":1789163197.0682123,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","EoS":false} {"level":"info","ts":1789163197.0682218,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"info","ts":1789163197.0682473,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf","EoS":true} {"level":"info","ts":1789163197.068262,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3a9624a9ce87eec7bf64d5466d24ca52","span_id":"80f40c32d62dc62f","x-request-id":"0d3a2409-0e43-48e3-9b88-48a7eab582cf"} {"level":"error","ts":1789163207.3675601,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163540.6154964,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163540.61562,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.6156354,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.615654,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","EoS":true} {"level":"info","ts":1789163540.6156921,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","header":"x-maas-subscription"} {"level":"info","ts":1789163540.615705,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","header":"x-maas-username"} {"level":"info","ts":1789163540.615713,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","header":"x-maas-group"} {"level":"info","ts":1789163540.615736,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163540.6157706,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.6284883,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.6285136,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.6285348,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","EoS":false} {"level":"info","ts":1789163540.6285474,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.6285691,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987","EoS":true} {"level":"info","ts":1789163540.6285758,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cd3aa3a0e3561128bd60c069d8a65f03","span_id":"0bfede823ee8bb72","x-request-id":"cb8ba876-1b90-49a7-b328-061ecb8d2987"} {"level":"info","ts":1789163540.7728062,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163540.7729392,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.7729764,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.7729914,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","EoS":true} {"level":"info","ts":1789163540.7730231,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","header":"x-maas-subscription"} {"level":"info","ts":1789163540.7730343,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","header":"x-maas-username"} {"level":"info","ts":1789163540.7730389,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","header":"x-maas-group"} {"level":"info","ts":1789163540.7730534,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163540.773081,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.775205,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.7752266,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.7752416,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","EoS":false} {"level":"info","ts":1789163540.7752535,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.775279,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce","EoS":true} {"level":"info","ts":1789163540.7752888,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2267a4b4a14c76d91fa3fce10360f865","span_id":"39fb258aaa8a2971","x-request-id":"504672ac-a668-450e-8c99-84b3aa640fce"} {"level":"info","ts":1789163540.916764,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163540.9169066,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163540.9169252,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163540.916943,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","EoS":true} {"level":"info","ts":1789163540.916977,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","header":"x-maas-group"} {"level":"info","ts":1789163540.9169908,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","header":"x-maas-subscription"} {"level":"info","ts":1789163540.9169977,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","header":"x-maas-username"} {"level":"info","ts":1789163540.9170187,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163540.917051,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163540.9179895,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163540.9180086,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163540.9180243,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","EoS":false} {"level":"info","ts":1789163540.9180365,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163540.9180572,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f","EoS":true} {"level":"info","ts":1789163540.9180682,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"800ea6afcfa8fbc392bc93968abc6664","span_id":"3385e241133edc21","x-request-id":"97a38479-982d-4717-bad9-02134ab80e4f"} {"level":"info","ts":1789163541.0566492,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163541.0567417,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.0567553,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.0567663,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","EoS":true} {"level":"info","ts":1789163541.0567877,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","header":"x-maas-username"} {"level":"info","ts":1789163541.0567987,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","header":"x-maas-subscription"} {"level":"info","ts":1789163541.0568035,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","header":"x-maas-group"} {"level":"info","ts":1789163541.056818,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163541.0569506,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.0577676,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.0577948,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.0578098,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","EoS":false} {"level":"info","ts":1789163541.057844,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.057872,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286","EoS":true} {"level":"info","ts":1789163541.057882,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fc55a79165701064a44c50f5b606e901","span_id":"b0fb6f8ee476f11c","x-request-id":"d8e2f224-da71-4ed1-a506-05aafa892286"} {"level":"info","ts":1789163541.1972775,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163541.1973839,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"info","ts":1789163541.1973984,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"info","ts":1789163541.197411,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","EoS":true} {"level":"info","ts":1789163541.197439,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","header":"x-maas-subscription"} {"level":"info","ts":1789163541.1974485,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","header":"x-maas-username"} {"level":"info","ts":1789163541.1974535,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","header":"x-maas-group"} {"level":"info","ts":1789163541.1974688,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163541.1974957,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"info","ts":1789163541.1984963,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"info","ts":1789163541.198515,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"info","ts":1789163541.1985276,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","EoS":false} {"level":"info","ts":1789163541.1985378,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"info","ts":1789163541.1985605,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f","EoS":true} {"level":"info","ts":1789163541.198574,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c1cc67207f8d857f29e66aaf4a81cdda","span_id":"ecc2c11ecb8ab8db","x-request-id":"954ab745-7c41-471e-8801-a2ae95db5d1f"} {"level":"error","ts":1789163552.4089367,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163553.1721284,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163553.172268,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.1722884,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.1723046,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","EoS":true} {"level":"info","ts":1789163553.172332,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","header":"x-maas-username"} {"level":"info","ts":1789163553.172343,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","header":"x-maas-subscription"} {"level":"info","ts":1789163553.1723506,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","header":"x-maas-group"} {"level":"info","ts":1789163553.1723661,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163553.1723938,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.1739492,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.1739695,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.1739836,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","EoS":false} {"level":"info","ts":1789163553.173996,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.1740258,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59","EoS":true} {"level":"info","ts":1789163553.1740358,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b1ce4283adf32607d791b371901e7634","span_id":"cad8ec2546209d99","x-request-id":"5a007b04-3b7e-4ebc-9ff2-5988f9dc9c59"} {"level":"info","ts":1789163553.2792292,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163553.2793524,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a"} {"level":"info","ts":1789163553.2802508,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a"} {"level":"info","ts":1789163553.2802804,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a"} {"level":"info","ts":1789163553.2802982,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a","EoS":false} {"level":"info","ts":1789163553.28031,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a"} {"level":"info","ts":1789163553.2803354,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a","EoS":true} {"level":"info","ts":1789163553.280346,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b0ac8e04f89e84d97d9361a3a7c484d8","span_id":"db47229a10873580","x-request-id":"38855ebc-53c3-487a-9e24-3d0fa644c88a"} {"level":"info","ts":1789163554.0428903,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163554.0429876,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163554.043001,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163554.0430129,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","EoS":true} {"level":"info","ts":1789163554.0430408,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","header":"x-maas-subscription"} {"level":"info","ts":1789163554.0430515,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","header":"x-maas-group"} {"level":"info","ts":1789163554.0430565,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","header":"x-maas-username"} {"level":"info","ts":1789163554.0430703,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163554.0430968,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163554.043852,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163554.0438745,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163554.0438976,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","EoS":false} {"level":"info","ts":1789163554.0439112,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163554.0439413,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54","EoS":true} {"level":"info","ts":1789163554.0439515,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c2679c4e99f60466c07fb8acd8a3b014","span_id":"9040467c83f17e55","x-request-id":"235e6ad9-9eba-4c0e-a63f-9d24a39d0a54"} {"level":"info","ts":1789163555.0103548,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163555.0104592,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.010471,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.0104833,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","EoS":true} {"level":"info","ts":1789163555.0105069,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","header":"x-maas-subscription"} {"level":"info","ts":1789163555.0105186,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","header":"x-maas-username"} {"level":"info","ts":1789163555.0105233,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","header":"x-maas-group"} {"level":"info","ts":1789163555.01054,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163555.0105646,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.011337,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.0113583,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.011372,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","EoS":false} {"level":"info","ts":1789163555.0113833,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.011405,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90","EoS":true} {"level":"info","ts":1789163555.011413,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6cc49bacdcf25e4cc9aea2fc953c1245","span_id":"e1cd5f747089cc18","x-request-id":"8f0d6dd8-3544-466e-90ab-fff3ad7dfb90"} {"level":"info","ts":1789163555.7546394,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163555.7547455,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163555.7547584,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163555.7547712,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","EoS":true} {"level":"info","ts":1789163555.7548015,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","header":"x-maas-subscription"} {"level":"info","ts":1789163555.7548118,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","header":"x-maas-username"} {"level":"info","ts":1789163555.7548177,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","header":"x-maas-group"} {"level":"info","ts":1789163555.754853,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163555.7548819,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163555.755619,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163555.7556384,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163555.75565,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","EoS":false} {"level":"info","ts":1789163555.7556624,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163555.7556834,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71","EoS":true} {"level":"info","ts":1789163555.7556913,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"850eabda7264e35044137ef0f21363ce","span_id":"5933169bd4c57d7c","x-request-id":"ca33901e-8560-43bd-ba55-5695251cdf71"} {"level":"info","ts":1789163556.8273034,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163556.8274214,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"info","ts":1789163556.8274355,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"info","ts":1789163556.8274539,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","EoS":true} {"level":"info","ts":1789163556.8274932,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","header":"x-maas-group"} {"level":"info","ts":1789163556.82751,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","header":"x-maas-username"} {"level":"info","ts":1789163556.8275177,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","header":"x-maas-subscription"} {"level":"info","ts":1789163556.8275402,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163556.8275754,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"info","ts":1789163556.8282857,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"info","ts":1789163556.8283062,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"info","ts":1789163556.828324,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","EoS":false} {"level":"info","ts":1789163556.8283362,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"info","ts":1789163556.828362,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958","EoS":true} {"level":"info","ts":1789163556.8283722,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d9acd0d69308e975fbb1d102db502005","span_id":"26dd6aa16379af93","x-request-id":"3756133d-6538-4b72-8761-8a5d0ef1c958"} {"level":"error","ts":1789163567.4102943,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163574.3345468,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163574.3346558,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"info","ts":1789163574.3346703,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"info","ts":1789163574.3346853,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","EoS":true} {"level":"info","ts":1789163574.3347158,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","header":"x-maas-subscription"} {"level":"info","ts":1789163574.334726,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","header":"x-maas-group"} {"level":"info","ts":1789163574.3347313,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","header":"x-maas-username"} {"level":"info","ts":1789163574.3347461,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163574.3347733,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"info","ts":1789163574.335481,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"info","ts":1789163574.3355002,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"info","ts":1789163574.3355148,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","EoS":false} {"level":"info","ts":1789163574.3355265,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"info","ts":1789163574.3355517,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e","EoS":true} {"level":"info","ts":1789163574.3355615,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"22980dfca39133016b23183b9d68a4f5","span_id":"b122ddcad3bad892","x-request-id":"eec512e8-4f8a-4a11-95e5-12bfd17eac1e"} {"level":"error","ts":1789163587.4122,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163661.4015007,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163661.4016063,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"info","ts":1789163661.401619,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"info","ts":1789163661.4016302,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","EoS":true} {"level":"info","ts":1789163661.4016535,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","header":"x-maas-username"} {"level":"info","ts":1789163661.401663,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","header":"x-maas-subscription"} {"level":"info","ts":1789163661.401668,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","header":"x-maas-group"} {"level":"info","ts":1789163661.401684,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789163661.4017143,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"info","ts":1789163661.4140122,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"info","ts":1789163661.4140377,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"info","ts":1789163661.4140537,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","EoS":false} {"level":"info","ts":1789163661.4140675,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"info","ts":1789163661.4140947,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b","EoS":true} {"level":"info","ts":1789163661.4141018,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c898a226dedfd4eb606e630afa3f5496","span_id":"b44988bd4ecbc277","x-request-id":"edc1ef29-e0cc-4d8f-95a5-f707cc84712b"} {"level":"error","ts":1789163672.4206617,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163712.87845,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163712.878564,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163712.8785815,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163712.878595,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","EoS":true} {"level":"info","ts":1789163712.8786209,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","header":"x-maas-username"} {"level":"info","ts":1789163712.878631,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","header":"x-maas-group"} {"level":"info","ts":1789163712.8786366,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","header":"x-maas-subscription"} {"level":"info","ts":1789163712.8786514,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789163712.8786767,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163712.8795552,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163712.8795757,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163712.8795927,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","EoS":false} {"level":"info","ts":1789163712.879608,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163712.8796287,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074","EoS":true} {"level":"info","ts":1789163712.879638,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"118c479fc467d44a7b65e2dbb743ac66","span_id":"3c1e7803c3723a57","x-request-id":"11032ff2-90a9-47ec-a44c-541325910074"} {"level":"info","ts":1789163726.5858128,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163726.5859804,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.5860033,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.5860229,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","EoS":true} {"level":"info","ts":1789163726.5860577,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","header":"x-maas-username"} {"level":"info","ts":1789163726.5860732,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","header":"x-maas-subscription"} {"level":"info","ts":1789163726.5860803,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","header":"x-maas-group"} {"level":"info","ts":1789163726.5860982,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789163726.586126,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.599205,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.5992217,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.599236,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","EoS":false} {"level":"info","ts":1789163726.5992498,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.599273,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","EoS":false} {"level":"info","ts":1789163726.5992815,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.5992918,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7","EoS":true} {"level":"info","ts":1789163726.5993032,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01ded1249aed2cf93732210401463528","span_id":"eceab0135006abe5","x-request-id":"c025a17b-858f-43a0-be10-22d533b414f7"} {"level":"info","ts":1789163726.740142,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163726.7402716,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.740288,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.7403018,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","EoS":true} {"level":"info","ts":1789163726.740327,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","header":"x-maas-group"} {"level":"info","ts":1789163726.7403371,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","header":"x-maas-subscription"} {"level":"info","ts":1789163726.7403424,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","header":"x-maas-username"} {"level":"info","ts":1789163726.7403584,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789163726.740388,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.7427719,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.7427912,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.7428095,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","EoS":false} {"level":"info","ts":1789163726.7428563,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.742891,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","EoS":false} {"level":"info","ts":1789163726.7429094,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.743,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","EoS":false} {"level":"info","ts":1789163726.743025,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.7430463,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8","EoS":true} {"level":"info","ts":1789163726.7430553,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"58fbb9db3f580fea1a09eb311ecae882","span_id":"add683fcb84e4ad8","x-request-id":"3550c0f6-4c57-4503-b2f3-5a1bd07308f8"} {"level":"info","ts":1789163726.882586,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163726.8827019,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.8827193,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.8827312,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","EoS":true} {"level":"info","ts":1789163726.8827522,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","header":"x-maas-username"} {"level":"info","ts":1789163726.882763,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","header":"x-maas-group"} {"level":"info","ts":1789163726.8827682,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","header":"x-maas-subscription"} {"level":"info","ts":1789163726.8827872,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789163726.882815,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.8837724,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.8837905,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.883807,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","EoS":false} {"level":"info","ts":1789163726.8838427,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.883873,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","EoS":false} {"level":"info","ts":1789163726.8838863,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"info","ts":1789163726.8838973,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c","EoS":true} {"level":"info","ts":1789163726.8839028,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c963a44e46d2aa5e28b70bcf0b9b387","span_id":"cd24e6617581baf6","x-request-id":"228b8292-1a85-4bc1-9fb4-e0f4870beb6c"} {"level":"error","ts":1789163727.4266276,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789163739.7376397,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789163739.737757,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"info","ts":1789163739.7377787,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"info","ts":1789163739.7377992,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","EoS":true} {"level":"info","ts":1789163739.7378511,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","header":"x-maas-username"} {"level":"info","ts":1789163739.7378664,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","header":"x-maas-group"} {"level":"info","ts":1789163739.7378836,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","header":"x-maas-subscription"} {"level":"info","ts":1789163739.7379043,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789163739.7379377,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"info","ts":1789163739.739963,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"info","ts":1789163739.739986,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"info","ts":1789163739.7400029,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","EoS":false} {"level":"info","ts":1789163739.7400177,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"info","ts":1789163739.740049,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615","EoS":true} {"level":"info","ts":1789163739.7400627,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"93f89a5ddc4fdec697f2797f06325ca1","span_id":"c9140f027e90a696","x-request-id":"3fd3fc67-f977-4511-b647-c86d54ce8615"} {"level":"error","ts":1789163742.4279094,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}