{"level":"info","ts":1789566349.291526,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"f13f9ce9-5aba-4428-9238-c5ce886430e4","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.2915883,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"f13f9ce9-5aba-4428-9238-c5ce886430e4","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789566349.2917,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ae3a7fed-2768-4bc8-8a3d-85636e99e1df"} {"level":"error","ts":1789566349.2963457,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"d9628fee-c4ad-4562-8162-06cd0d08b2a3","error":"failed to apply ExternalProvider: externalproviders.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.29966,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ba5b1987-544a-4ce6-9981-410eb1549746"} {"level":"info","ts":1789566349.2999723,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"0ceefed7-9245-4588-8aec-4bcbe7617a6b","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.300004,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"0ceefed7-9245-4588-8aec-4bcbe7617a6b","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"info","ts":1789566349.303204,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"4124e1f9-4c9c-4841-b2b3-5e6333b55e0f","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3037772,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"9bba9a8b-eefc-4d6d-9495-dcc4e0a4fd38","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3038037,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"9bba9a8b-eefc-4d6d-9495-dcc4e0a4fd38","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789566349.3039246,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"2cb6c633-9e1d-44ec-a7a1-17f2c382c7f7","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.303947,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"2cb6c633-9e1d-44ec-a7a1-17f2c382c7f7","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"error","ts":1789566349.3042715,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ae3a7fed-2768-4bc8-8a3d-85636e99e1df","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:84\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.3043408,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ae3a7fed-2768-4bc8-8a3d-85636e99e1df","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3066063,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"3573904b-f7f5-4168-8a83-d522ef57ed36","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3066413,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"3573904b-f7f5-4168-8a83-d522ef57ed36","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"error","ts":1789566349.3112533,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ba5b1987-544a-4ce6-9981-410eb1549746","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.3113074,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ba5b1987-544a-4ce6-9981-410eb1549746","provider":"e2e-legacy-migration-aeb95b0c","phase":"Pending"} {"level":"info","ts":1789566349.3144405,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"4124e1f9-4c9c-4841-b2b3-5e6333b55e0f","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3152854,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"88c351e9-422d-40c8-9b52-f1225f42e552","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.31537,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"88c351e9-422d-40c8-9b52-f1225f42e552","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"error","ts":1789566349.3191454,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ba5b1987-544a-4ce6-9981-410eb1549746","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789566349.319191,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ba5b1987-544a-4ce6-9981-410eb1549746","error":"ExternalModel \"e2e-legacy-migration-aeb95b0c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-aeb95b0c\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.3192391,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"baacd143-71d1-490e-bff0-839fda19651a"} {"level":"info","ts":1789566349.319268,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"baacd143-71d1-490e-bff0-839fda19651a","provider":"e2e-legacy-migration-aeb95b0c","phase":"Pending"} {"level":"error","ts":1789566349.3206866,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"4124e1f9-4c9c-4841-b2b3-5e6333b55e0f","error":"failed to apply ExternalModel: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789566349.3256152,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"baacd143-71d1-490e-bff0-839fda19651a","error":"ExternalModel \"e2e-legacy-migration-aeb95b0c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-aeb95b0c\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789566349.3260617,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ae3a7fed-2768-4bc8-8a3d-85636e99e1df","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789566349.3261132,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ae3a7fed-2768-4bc8-8a3d-85636e99e1df","error":"failed to apply Service: services \"e2e-legacy-migration-aeb95b0c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.32616,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c79b78e5-b52a-4c84-a205-5fc702767456"} {"level":"info","ts":1789566349.3261998,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c79b78e5-b52a-4c84-a205-5fc702767456","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3310115,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ec242c51-9627-48c0-bc79-2a2240cb6bc7","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3365176,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"870f8367-8896-4c8a-bdf2-80a3518329b5"} {"level":"info","ts":1789566349.3365421,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"870f8367-8896-4c8a-bdf2-80a3518329b5","provider":"e2e-legacy-migration-aeb95b0c","phase":"Pending"} {"level":"info","ts":1789566349.3380733,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ec242c51-9627-48c0-bc79-2a2240cb6bc7","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3394656,"caller":"externalprovider/reconciler.go:210","msg":"Creating resource","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c79b78e5-b52a-4c84-a205-5fc702767456","kind":"DestinationRule","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3498282,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ec242c51-9627-48c0-bc79-2a2240cb6bc7","provider":"openai","model":"e2e-legacy-migration-aeb95b0c","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789566349.3498607,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"db3b2df3-f58d-4d35-9a3f-11b31f822d9a","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3499007,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"db3b2df3-f58d-4d35-9a3f-11b31f822d9a","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789566349.349939,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"3cacb20e-c757-4c26-aced-50e3827a3b49","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.349967,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"3cacb20e-c757-4c26-aced-50e3827a3b49","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"error","ts":1789566349.35023,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"870f8367-8896-4c8a-bdf2-80a3518329b5","error":"ExternalModel \"e2e-legacy-migration-aeb95b0c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-aeb95b0c\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.3502808,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"dd357832-e5c0-4dbf-8ad4-1a888660b0be"} {"level":"info","ts":1789566349.3503196,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"dd357832-e5c0-4dbf-8ad4-1a888660b0be","name":"e2e-legacy-migration-aeb95b0c"} {"level":"error","ts":1789566349.3546746,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c79b78e5-b52a-4c84-a205-5fc702767456","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789566349.3547113,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c79b78e5-b52a-4c84-a205-5fc702767456","error":"failed to apply DestinationRule: destinationrules.networking.istio.io \"e2e-legacy-migration-aeb95b0c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.3547492,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c5d25395-c523-414d-8f84-20e653ca33dd"} {"level":"info","ts":1789566349.3547833,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c5d25395-c523-414d-8f84-20e653ca33dd","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3723855,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"a36bc24e-a5ea-4238-bedb-6ae1e5672338","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3724134,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"a36bc24e-a5ea-4238-bedb-6ae1e5672338","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"info","ts":1789566349.3767874,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"c5d25395-c523-414d-8f84-20e653ca33dd","service":"e2e-legacy-migration-aeb95b0c","serviceEntry":"e2e-legacy-migration-aeb95b0c","destinationRule":"e2e-legacy-migration-aeb95b0c"} {"level":"error","ts":1789566349.3809931,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"dd357832-e5c0-4dbf-8ad4-1a888660b0be","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789566349.3810823,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"dd357832-e5c0-4dbf-8ad4-1a888660b0be","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-aeb95b0c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.3812153,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"5b948351-74fe-4cfc-940f-a40b4abf60af"} {"level":"info","ts":1789566349.3813024,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"5b948351-74fe-4cfc-940f-a40b4abf60af","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.3930612,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"44b670d4-7669-4adc-bf1b-f14231365c67","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.3931808,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"44b670d4-7669-4adc-bf1b-f14231365c67","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"info","ts":1789566349.4005742,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"5b948351-74fe-4cfc-940f-a40b4abf60af","httpRoute":"e2e-legacy-migration-aeb95b0c","providers":["e2e-legacy-migration-aeb95b0c"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789566349.4085739,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"5b948351-74fe-4cfc-940f-a40b4abf60af","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.4086895,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ce8a31fd-57f6-4683-b48c-8dd5a7d6fab5"} {"level":"info","ts":1789566349.4087644,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ce8a31fd-57f6-4683-b48c-8dd5a7d6fab5","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.4132674,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"6d7cb4c0-5ded-4cfa-9d2e-34e522282117","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.4133089,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"6d7cb4c0-5ded-4cfa-9d2e-34e522282117","modelName":"e2e-legacy-migration-aeb95b0c","resolvedRefs":1} {"level":"info","ts":1789566349.4207997,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ce8a31fd-57f6-4683-b48c-8dd5a7d6fab5","httpRoute":"e2e-legacy-migration-aeb95b0c","providers":["e2e-legacy-migration-aeb95b0c"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789566349.4321973,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"ce8a31fd-57f6-4683-b48c-8dd5a7d6fab5","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-aeb95b0c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789566349.432283,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"2b232d85-4454-419d-8a25-3c25be482295"} {"level":"info","ts":1789566349.4323416,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"2b232d85-4454-419d-8a25-3c25be482295","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.4464455,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"2b232d85-4454-419d-8a25-3c25be482295","httpRoute":"e2e-legacy-migration-aeb95b0c","providers":["e2e-legacy-migration-aeb95b0c"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789566349.4915833,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"54a13cf6-834f-42fc-85cc-1bebf6e17358","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566349.5074553,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"54a13cf6-834f-42fc-85cc-1bebf6e17358","name":"e2e-legacy-migration-aeb95b0c"} {"level":"info","ts":1789566349.5134382,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"54a13cf6-834f-42fc-85cc-1bebf6e17358","provider":"openai","model":"e2e-legacy-migration-aeb95b0c","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789566352.2331634,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"1d483d94-1731-490f-9510-ff06ce05c2a2","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566352.247169,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"901cce61-16e7-4cb8-a7d0-e23f1e5d7a82"} {"level":"info","ts":1789566352.2472022,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"dfad84c7-b8a9-498d-b2a6-af91500363ae"} {"level":"info","ts":1789566352.247252,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"8321f912-0717-4c8c-9f6b-6a4455e69f2e","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566352.2472692,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"8321f912-0717-4c8c-9f6b-6a4455e69f2e","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566352.2472816,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"f3ecd07d-bfad-41de-b520-7536962e49dc","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566352.2473013,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-aeb95b0c","reconcileID":"f3ecd07d-bfad-41de-b520-7536962e49dc","name":"e2e-legacy-migration-aeb95b0c","namespace":"llm"} {"level":"info","ts":1789566352.6842651,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789566352.6843708,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566352.684389,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566352.6844018,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","EoS":true} {"level":"info","ts":1789566352.6844249,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","header":"x-maas-username"} {"level":"info","ts":1789566352.6844347,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","header":"x-maas-subscription"} {"level":"info","ts":1789566352.68444,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","header":"x-maas-keyname"} {"level":"info","ts":1789566352.6844442,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","header":"x-maas-group"} {"level":"info","ts":1789566352.6844635,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789566352.684477,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","modelName":"e2e-external-model"} {"level":"info","ts":1789566352.6844873,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789566352.68451,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789566352.6845255,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","authType":"apikey"} {"level":"info","ts":1789566352.6845577,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566353.453797,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566353.453826,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566353.4538426,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","EoS":false} {"level":"info","ts":1789566353.4538555,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566353.4538882,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428","EoS":true} {"level":"info","ts":1789566353.4538987,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b3236ea5fc0af1877eb285767c6910ba","span_id":"d9a9a4cfdc2e3248","x-request-id":"b862da77-3491-4ca9-9225-f15c915c4428"} {"level":"info","ts":1789566353.5064437,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789566353.50656,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566353.5065765,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566353.5065897,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","EoS":true} {"level":"info","ts":1789566353.5066166,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","header":"x-maas-group"} {"level":"info","ts":1789566353.506627,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","header":"x-maas-username"} {"level":"info","ts":1789566353.506635,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","header":"x-maas-keyname"} {"level":"info","ts":1789566353.5066435,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","header":"x-maas-subscription"} {"level":"info","ts":1789566353.5066607,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789566353.5066926,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566354.645155,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566354.6451924,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566354.6452162,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","EoS":false} {"level":"info","ts":1789566354.6452346,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566354.645275,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb","EoS":true} {"level":"info","ts":1789566354.64529,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8043549036fe2b4f74ae07b004054199","span_id":"842cc651c5ee9f73","x-request-id":"1cd9086a-908e-449b-8cf8-709ce8bf50fb"} {"level":"info","ts":1789566354.7034342,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789566354.7035675,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"info","ts":1789566354.703589,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"info","ts":1789566354.7036037,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","EoS":true} {"level":"info","ts":1789566354.7036347,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","header":"x-maas-username"} {"level":"info","ts":1789566354.7036424,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","header":"x-maas-group"} {"level":"info","ts":1789566354.7036502,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","header":"x-maas-subscription"} {"level":"info","ts":1789566354.7036602,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","header":"x-maas-keyname"} {"level":"info","ts":1789566354.7036972,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"error","ts":1789566355.2039466,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789566356.5008028,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"info","ts":1789566356.5008326,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"info","ts":1789566356.5008514,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","EoS":false} {"level":"info","ts":1789566356.5008688,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"info","ts":1789566356.5009072,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9","EoS":true} {"level":"info","ts":1789566356.5009186,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"eb035c0e1913fa169dc5aa219a657487","span_id":"19161d8e8096e813","x-request-id":"a5787ab3-ae1c-4d3b-8b27-b55941fcd7a9"} {"level":"info","ts":1789566357.3674405,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"38e14870-d998-4564-81fb-a943fbbc0b93"} {"level":"info","ts":1789566357.3674846,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"ab699bb4-00c5-4a77-bf1c-173267bce654","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789566357.3675237,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"ab699bb4-00c5-4a77-bf1c-173267bce654","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789566357.501909,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"44274c00-7d85-48fb-8d12-d670f8b4ccb7","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789566357.5019503,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"44274c00-7d85-48fb-8d12-d670f8b4ccb7","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789566357.5019448,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"0630813d-dba5-4836-8b9f-82fcf89ff2ca"} {"level":"info","ts":1789566357.6122963,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"001ff1ba-8fdf-4ef6-b3ff-b6c774ea082d","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789566357.612362,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"001ff1ba-8fdf-4ef6-b3ff-b6c774ea082d","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789566361.320693,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789566361.3208025,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"196d76e2e103bbc01d53cc4616c5ba4d","span_id":"ce922b59923da817","x-request-id":"fcea14c3-576c-4a76-aa13-5b778c514f71"} {"level":"info","ts":1789566361.3208191,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"196d76e2e103bbc01d53cc4616c5ba4d","span_id":"ce922b59923da817","x-request-id":"fcea14c3-576c-4a76-aa13-5b778c514f71"} {"level":"info","ts":1789566361.320835,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"196d76e2e103bbc01d53cc4616c5ba4d","span_id":"ce922b59923da817","x-request-id":"fcea14c3-576c-4a76-aa13-5b778c514f71","EoS":true} {"level":"info","ts":1789566361.3208888,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"196d76e2e103bbc01d53cc4616c5ba4d","span_id":"ce922b59923da817","x-request-id":"fcea14c3-576c-4a76-aa13-5b778c514f71"} {"level":"info","ts":1789566361.4797778,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789566361.4798906,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9ba8369ad3e10514b220f98eb621fe5a","span_id":"3cd5871f46cf29b2","x-request-id":"8e0ec03c-6be2-4569-acb5-7e3605649c74"} {"level":"info","ts":1789566361.4799058,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9ba8369ad3e10514b220f98eb621fe5a","span_id":"3cd5871f46cf29b2","x-request-id":"8e0ec03c-6be2-4569-acb5-7e3605649c74"} {"level":"info","ts":1789566361.4799268,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9ba8369ad3e10514b220f98eb621fe5a","span_id":"3cd5871f46cf29b2","x-request-id":"8e0ec03c-6be2-4569-acb5-7e3605649c74","EoS":true} {"level":"info","ts":1789566361.4799824,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9ba8369ad3e10514b220f98eb621fe5a","span_id":"3cd5871f46cf29b2","x-request-id":"8e0ec03c-6be2-4569-acb5-7e3605649c74"} {"level":"info","ts":1789566361.6492116,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789566361.6493216,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"24000f9e8cb3508a400fb8d100e76b5b","span_id":"34cd1e019db9281a","x-request-id":"4c3a4ef1-58d7-4c41-9950-7a45da16d434"} {"level":"info","ts":1789566361.6493351,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"24000f9e8cb3508a400fb8d100e76b5b","span_id":"34cd1e019db9281a","x-request-id":"4c3a4ef1-58d7-4c41-9950-7a45da16d434"} {"level":"info","ts":1789566361.6493468,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"24000f9e8cb3508a400fb8d100e76b5b","span_id":"34cd1e019db9281a","x-request-id":"4c3a4ef1-58d7-4c41-9950-7a45da16d434","EoS":true} {"level":"info","ts":1789566361.6494036,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"24000f9e8cb3508a400fb8d100e76b5b","span_id":"34cd1e019db9281a","x-request-id":"4c3a4ef1-58d7-4c41-9950-7a45da16d434"} {"level":"error","ts":1789566370.2043154,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789566380.2052171,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789566395.20682,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567397.1540525,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567397.154214,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1542323,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1542468,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","EoS":true} {"level":"info","ts":1789567397.1542737,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","header":"x-maas-keyname"} {"level":"info","ts":1789567397.1542878,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","header":"x-maas-username"} {"level":"info","ts":1789567397.1542945,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","header":"x-maas-group"} {"level":"info","ts":1789567397.1542997,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","header":"x-maas-subscription"} {"level":"info","ts":1789567397.1543162,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789567397.1543484,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1560266,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1560473,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1560638,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","EoS":false} {"level":"info","ts":1789567397.156079,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1561341,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd","EoS":true} {"level":"info","ts":1789567397.1561446,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e38cd64e99bcf5b7c95062f140de7963","span_id":"8336879b60a39fd4","x-request-id":"abd789e1-3745-4e3b-9af4-fa1c39612cfd"} {"level":"info","ts":1789567397.1930134,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567397.1931555,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"info","ts":1789567397.193172,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"info","ts":1789567397.1931853,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","EoS":true} {"level":"info","ts":1789567397.193211,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","header":"x-maas-subscription"} {"level":"info","ts":1789567397.1932218,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","header":"x-maas-group"} {"level":"info","ts":1789567397.1932275,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","header":"x-maas-keyname"} {"level":"info","ts":1789567397.1932325,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","header":"x-maas-username"} {"level":"info","ts":1789567397.1932485,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789567397.1932783,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"info","ts":1789567397.194281,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"info","ts":1789567397.194301,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"info","ts":1789567397.1943219,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","EoS":false} {"level":"info","ts":1789567397.1943371,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"info","ts":1789567397.1943688,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc","EoS":true} {"level":"info","ts":1789567397.194393,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"018de580c37691f7ef1e9644235f3f47","span_id":"84a4e739b0029e28","x-request-id":"13b839a0-bb7b-401d-bfcf-5d94a1aaa7fc"} {"level":"error","ts":1789567410.3227265,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567709.4750524,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567709.475171,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.4751832,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.4751952,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","EoS":true} {"level":"info","ts":1789567709.475218,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","header":"x-maas-username"} {"level":"info","ts":1789567709.4752288,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","header":"x-maas-keyname"} {"level":"info","ts":1789567709.4752338,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","header":"x-maas-subscription"} {"level":"info","ts":1789567709.475238,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","header":"x-maas-group"} {"level":"info","ts":1789567709.475255,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567709.4752836,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.4877734,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.4878,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.4878151,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","EoS":false} {"level":"info","ts":1789567709.4878273,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.487847,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7","EoS":true} {"level":"info","ts":1789567709.487856,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b10c4a4a39f993af4ed33f836e660322","span_id":"0b35a44f03244c87","x-request-id":"75d9d7be-a95b-4285-ba12-7d5e2b360cf7"} {"level":"info","ts":1789567709.627965,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567709.6280718,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.6281016,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.628115,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","EoS":true} {"level":"info","ts":1789567709.628138,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","header":"x-maas-username"} {"level":"info","ts":1789567709.6281483,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","header":"x-maas-group"} {"level":"info","ts":1789567709.6281533,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","header":"x-maas-keyname"} {"level":"info","ts":1789567709.6281579,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","header":"x-maas-subscription"} {"level":"info","ts":1789567709.6281726,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567709.628199,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.6290689,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.629082,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.6291196,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","EoS":false} {"level":"info","ts":1789567709.6291306,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.6291542,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6","EoS":true} {"level":"info","ts":1789567709.629167,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b9b66a1ebb0f619e828c3ab7c76f0ac7","span_id":"8ed62cd6891c036a","x-request-id":"b74b3172-0b04-410f-ab7d-99fba8c32bb6"} {"level":"info","ts":1789567709.7744002,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567709.7745004,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.7745123,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.774524,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","EoS":true} {"level":"info","ts":1789567709.7745492,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","header":"x-maas-group"} {"level":"info","ts":1789567709.7745597,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","header":"x-maas-keyname"} {"level":"info","ts":1789567709.7745667,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","header":"x-maas-subscription"} {"level":"info","ts":1789567709.774571,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","header":"x-maas-username"} {"level":"info","ts":1789567709.7745862,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567709.7746115,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.7772577,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.777283,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.7773001,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","EoS":false} {"level":"info","ts":1789567709.7773147,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.7773442,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890","EoS":true} {"level":"info","ts":1789567709.7773561,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d94d42c29d68b144b65feca0d3a2a0ac","span_id":"0fdf714eac4432fb","x-request-id":"68635518-740a-4b4c-9911-603750710890"} {"level":"info","ts":1789567709.9143903,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567709.9145062,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567709.9145226,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567709.9145358,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","EoS":true} {"level":"info","ts":1789567709.9145622,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","header":"x-maas-group"} {"level":"info","ts":1789567709.9145725,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","header":"x-maas-subscription"} {"level":"info","ts":1789567709.9145775,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","header":"x-maas-keyname"} {"level":"info","ts":1789567709.9145825,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","header":"x-maas-username"} {"level":"info","ts":1789567709.914597,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567709.9146268,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567709.9155128,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567709.9155333,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567709.9155498,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","EoS":false} {"level":"info","ts":1789567709.9155614,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567709.9155886,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199","EoS":true} {"level":"info","ts":1789567709.9156046,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c975a27b83bbf7559f9f910377b3355e","span_id":"c4f6562babd8fba6","x-request-id":"1d7b642e-7dd1-4094-b65c-37ebaef87199"} {"level":"info","ts":1789567710.0558448,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567710.0559692,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567710.055987,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567710.0560014,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","EoS":true} {"level":"info","ts":1789567710.0560253,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","header":"x-maas-group"} {"level":"info","ts":1789567710.0560334,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","header":"x-maas-keyname"} {"level":"info","ts":1789567710.0560386,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","header":"x-maas-subscription"} {"level":"info","ts":1789567710.0560436,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","header":"x-maas-username"} {"level":"info","ts":1789567710.0560577,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567710.0561109,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567710.057067,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567710.057108,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567710.0571272,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","EoS":false} {"level":"info","ts":1789567710.0571394,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567710.0571642,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1","EoS":true} {"level":"info","ts":1789567710.0571742,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"56bf4b8f7cf414a0efdde76fb38e829d","span_id":"39849875b1b2bdbc","x-request-id":"3fca833e-6492-4932-b7b6-d77f68f324f1"} {"level":"info","ts":1789567718.8569927,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567718.85711,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.8571234,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.8571355,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","EoS":true} {"level":"info","ts":1789567718.85716,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","header":"x-maas-subscription"} {"level":"info","ts":1789567718.857171,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","header":"x-maas-username"} {"level":"info","ts":1789567718.8571754,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","header":"x-maas-keyname"} {"level":"info","ts":1789567718.8571794,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","header":"x-maas-group"} {"level":"info","ts":1789567718.857194,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567718.8572185,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.8581069,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.8581285,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.858144,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","EoS":false} {"level":"info","ts":1789567718.858156,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.8581765,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52","EoS":true} {"level":"info","ts":1789567718.8581884,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c7cf286b486e9f0e375971bc7abe9bda","span_id":"fd80b0546262cc80","x-request-id":"e19a31c8-1642-4e2c-8a29-3792c7746d52"} {"level":"info","ts":1789567718.8922975,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567718.892394,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.8924048,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.8924153,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","EoS":true} {"level":"info","ts":1789567718.892438,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","header":"x-maas-keyname"} {"level":"info","ts":1789567718.892446,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","header":"x-maas-subscription"} {"level":"info","ts":1789567718.8924513,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","header":"x-maas-username"} {"level":"info","ts":1789567718.8924563,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","header":"x-maas-group"} {"level":"info","ts":1789567718.8924704,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567718.892494,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.8933125,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.8933334,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.8933465,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","EoS":false} {"level":"info","ts":1789567718.8933578,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.8933837,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3","EoS":true} {"level":"info","ts":1789567718.8933918,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d954cdc4a03a9a0b9ffeb47c7c9623d3","span_id":"75cf3bd193312efc","x-request-id":"6e5bd1ee-0a38-4f75-9869-7063be61d9f3"} {"level":"info","ts":1789567718.998705,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567718.9988267,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324"} {"level":"info","ts":1789567718.9996626,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324"} {"level":"info","ts":1789567718.9996834,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324"} {"level":"info","ts":1789567718.9996982,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324","EoS":false} {"level":"info","ts":1789567718.999711,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324"} {"level":"info","ts":1789567718.999737,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324","EoS":true} {"level":"info","ts":1789567718.9997466,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c43519c3b4612f5f00fa0d711101d316","span_id":"c90a233560486e97","x-request-id":"ffd9c683-7f51-46f7-ab0f-f2394a440324"} {"level":"info","ts":1789567719.7800062,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567719.7801728,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"info","ts":1789567719.7801914,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"info","ts":1789567719.7802064,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","EoS":true} {"level":"info","ts":1789567719.7802296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","header":"x-maas-keyname"} {"level":"info","ts":1789567719.7802396,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","header":"x-maas-username"} {"level":"info","ts":1789567719.7802453,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","header":"x-maas-subscription"} {"level":"info","ts":1789567719.7802505,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","header":"x-maas-group"} {"level":"info","ts":1789567719.780266,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567719.7802927,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"info","ts":1789567719.7812622,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"info","ts":1789567719.7812827,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"info","ts":1789567719.7812943,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","EoS":false} {"level":"info","ts":1789567719.7813148,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"info","ts":1789567719.7813437,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f","EoS":true} {"level":"info","ts":1789567719.7813532,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6c85112da28fb14c9d1be93143527e99","span_id":"c356472fa453bb4a","x-request-id":"5cecb471-a1c7-4daa-9645-a1b633b2d20f"} {"level":"error","ts":1789567720.3545141,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567720.7076788,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567720.707782,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567720.7077951,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567720.707806,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","EoS":true} {"level":"info","ts":1789567720.707829,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","header":"x-maas-group"} {"level":"info","ts":1789567720.7078393,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","header":"x-maas-keyname"} {"level":"info","ts":1789567720.7078438,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","header":"x-maas-username"} {"level":"info","ts":1789567720.7078483,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","header":"x-maas-subscription"} {"level":"info","ts":1789567720.7078648,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567720.7078938,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567720.708909,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567720.708939,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567720.7089608,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","EoS":false} {"level":"info","ts":1789567720.7089791,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567720.7090259,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1","EoS":true} {"level":"info","ts":1789567720.7090418,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b5ddfce432640efa340cadfc372a83a4","span_id":"847a6ca6be397019","x-request-id":"75a4520e-ca56-4edc-9937-b843423137d1"} {"level":"info","ts":1789567721.4465733,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567721.446685,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567721.446702,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567721.4467177,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","EoS":true} {"level":"info","ts":1789567721.4467413,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","header":"x-maas-subscription"} {"level":"info","ts":1789567721.4467518,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","header":"x-maas-group"} {"level":"info","ts":1789567721.4467573,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","header":"x-maas-keyname"} {"level":"info","ts":1789567721.4467614,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","header":"x-maas-username"} {"level":"info","ts":1789567721.44678,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567721.4468143,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567721.447727,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567721.4477496,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567721.4477668,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","EoS":false} {"level":"info","ts":1789567721.4477777,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567721.4478052,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee","EoS":true} {"level":"info","ts":1789567721.4478178,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d35d2c01be32d7dae74546ba8f095936","span_id":"9be170b85f5a1248","x-request-id":"dd71a851-22f5-41e0-b371-b9e253a657ee"} {"level":"info","ts":1789567722.5158358,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567722.515945,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"info","ts":1789567722.5159633,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"info","ts":1789567722.5159767,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","EoS":true} {"level":"info","ts":1789567722.5160007,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","header":"x-maas-keyname"} {"level":"info","ts":1789567722.5160105,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","header":"x-maas-group"} {"level":"info","ts":1789567722.516017,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","header":"x-maas-username"} {"level":"info","ts":1789567722.5160239,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","header":"x-maas-subscription"} {"level":"info","ts":1789567722.5160437,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567722.5160787,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"info","ts":1789567722.5169623,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"info","ts":1789567722.51699,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"info","ts":1789567722.5170138,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","EoS":false} {"level":"info","ts":1789567722.517033,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"info","ts":1789567722.5170596,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5","EoS":true} {"level":"info","ts":1789567722.5170703,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0ff6591095060faec770a1e851901f25","span_id":"c5cac6f533356a18","x-request-id":"149aa4fd-723d-4167-8d41-413fa18074c5"} {"level":"error","ts":1789567735.3557546,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567742.1239104,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567742.1240115,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"info","ts":1789567742.1240253,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"info","ts":1789567742.1240368,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","EoS":true} {"level":"info","ts":1789567742.1240594,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","header":"x-maas-username"} {"level":"info","ts":1789567742.1240695,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","header":"x-maas-group"} {"level":"info","ts":1789567742.1240747,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","header":"x-maas-subscription"} {"level":"info","ts":1789567742.1240797,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","header":"x-maas-keyname"} {"level":"info","ts":1789567742.12412,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567742.1241457,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"info","ts":1789567742.1249764,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"info","ts":1789567742.1249979,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"info","ts":1789567742.1250196,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","EoS":false} {"level":"info","ts":1789567742.1250384,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"info","ts":1789567742.1250768,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03","EoS":true} {"level":"info","ts":1789567742.1251054,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4c22e4560cebe62594321eecea8c5da9","span_id":"bfb22247599b7a70","x-request-id":"4bd6e0f3-9788-4863-b0fc-6dabcb5ace03"} {"level":"error","ts":1789567755.3568401,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567833.7097423,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567833.709853,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"info","ts":1789567833.7098684,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"info","ts":1789567833.7098854,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","EoS":true} {"level":"info","ts":1789567833.709912,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","header":"x-maas-subscription"} {"level":"info","ts":1789567833.7099223,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","header":"x-maas-group"} {"level":"info","ts":1789567833.7099283,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","header":"x-maas-username"} {"level":"info","ts":1789567833.7099323,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","header":"x-maas-keyname"} {"level":"info","ts":1789567833.7099466,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789567833.7099805,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"info","ts":1789567833.7223463,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"info","ts":1789567833.722373,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"info","ts":1789567833.7223947,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","EoS":false} {"level":"info","ts":1789567833.7224128,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"info","ts":1789567833.7224543,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef","EoS":true} {"level":"info","ts":1789567833.7224662,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7131ce13d8e2ca6b26653607fc4c71c0","span_id":"acb9dd2aa9064c8a","x-request-id":"7f762c25-f6d3-4260-9847-fc73938037ef"} {"level":"error","ts":1789567845.366191,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567915.1603887,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567915.1605017,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"info","ts":1789567915.1605144,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"info","ts":1789567915.160526,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","EoS":true} {"level":"info","ts":1789567915.1605494,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","header":"x-maas-subscription"} {"level":"info","ts":1789567915.1605604,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","header":"x-maas-username"} {"level":"info","ts":1789567915.160565,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","header":"x-maas-keyname"} {"level":"info","ts":1789567915.1605701,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","header":"x-maas-group"} {"level":"info","ts":1789567915.1605859,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789567915.1606216,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"info","ts":1789567915.1622224,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"info","ts":1789567915.16225,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"info","ts":1789567915.1622713,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","EoS":false} {"level":"info","ts":1789567915.1622925,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"info","ts":1789567915.1623235,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636","EoS":true} {"level":"info","ts":1789567915.1623335,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5bc76c3d2ccd3c102d655ca4bc0f52a6","span_id":"d2b890f283b47d1a","x-request-id":"80f23fd7-fb32-4ece-8545-83c7da5ab636"} {"level":"error","ts":1789567925.3753347,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567929.1409237,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567929.1410418,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.141059,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.141072,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","EoS":true} {"level":"info","ts":1789567929.141123,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","header":"x-maas-username"} {"level":"info","ts":1789567929.1411357,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","header":"x-maas-keyname"} {"level":"info","ts":1789567929.1411414,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","header":"x-maas-group"} {"level":"info","ts":1789567929.1411462,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","header":"x-maas-subscription"} {"level":"info","ts":1789567929.1411617,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789567929.1411927,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.1550174,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.1550448,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.1550653,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","EoS":false} {"level":"info","ts":1789567929.1550813,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.155135,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","EoS":false} {"level":"info","ts":1789567929.1551528,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.1551723,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449","EoS":true} {"level":"info","ts":1789567929.1551864,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b6eeedba1be07d9c9a58187dd2ca774a","span_id":"1c76cdd7fd594fad","x-request-id":"5a0f4fd8-bdd1-4dff-acd8-cd17bfc94449"} {"level":"info","ts":1789567929.2959342,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567929.2960525,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.296071,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.2961075,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","EoS":true} {"level":"info","ts":1789567929.2961395,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","header":"x-maas-group"} {"level":"info","ts":1789567929.2961535,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","header":"x-maas-keyname"} {"level":"info","ts":1789567929.2961593,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","header":"x-maas-username"} {"level":"info","ts":1789567929.2961636,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","header":"x-maas-subscription"} {"level":"info","ts":1789567929.2961783,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789567929.2962046,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.29722,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.2972403,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.2972608,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","EoS":false} {"level":"info","ts":1789567929.2972765,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.297299,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","EoS":false} {"level":"info","ts":1789567929.297311,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.2973235,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027","EoS":true} {"level":"info","ts":1789567929.2973378,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4bd9ab21c5f9801458579534412f0f83","span_id":"76518d6e7ffe1859","x-request-id":"c46866ec-89d4-40f1-9651-547945fae027"} {"level":"info","ts":1789567929.4375038,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567929.437627,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.4376445,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.437658,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","EoS":true} {"level":"info","ts":1789567929.437682,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","header":"x-maas-keyname"} {"level":"info","ts":1789567929.4376929,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","header":"x-maas-subscription"} {"level":"info","ts":1789567929.4376981,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","header":"x-maas-group"} {"level":"info","ts":1789567929.4377027,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","header":"x-maas-username"} {"level":"info","ts":1789567929.437716,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789567929.4377444,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.438655,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.4386752,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.438696,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","EoS":false} {"level":"info","ts":1789567929.4387155,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.4387434,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","EoS":false} {"level":"info","ts":1789567929.4387543,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"info","ts":1789567929.4387653,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107","EoS":true} {"level":"info","ts":1789567929.4387722,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d0d6c8d3194ff939b1a97ed72f117e6","span_id":"b456d80e62c20a0c","x-request-id":"b48bdc79-481c-42d5-a5fb-fcff7225a107"} {"level":"error","ts":1789567940.3772483,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789567942.3878512,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789567942.3879564,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.387969,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.38798,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","EoS":true} {"level":"info","ts":1789567942.3880014,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","header":"x-maas-username"} {"level":"info","ts":1789567942.3880112,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","header":"x-maas-keyname"} {"level":"info","ts":1789567942.388017,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","header":"x-maas-subscription"} {"level":"info","ts":1789567942.3880217,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","header":"x-maas-group"} {"level":"info","ts":1789567942.3880374,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789567942.3880658,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.3901284,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.3901584,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.3901818,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","EoS":false} {"level":"info","ts":1789567942.3902016,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.390292,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","EoS":false} {"level":"info","ts":1789567942.3903074,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"info","ts":1789567942.3903353,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152","EoS":true} {"level":"info","ts":1789567942.3903475,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2ef628a03063007dc1ffda23120be838","span_id":"463a8a1667dde204","x-request-id":"b4ee8c39-968f-4280-bc86-1725953b3152"} {"level":"error","ts":1789567955.3784177,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}