{"level":"error","ts":1789147890.7486892,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"4aeae3e5-44a9-45eb-8c3f-4acf5d8afc30","error":"failed to apply Service: services \"e2e-legacy-migration-d1dc448d\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.748744,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8ec65326-da7b-4f23-87f2-d2719b172fdd"} {"level":"info","ts":1789147890.7487905,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8ec65326-da7b-4f23-87f2-d2719b172fdd","name":"e2e-legacy-migration-d1dc448d"} {"level":"error","ts":1789147890.7499049,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"fd026ed9-d3be-4caa-adc9-2d3f6e2fab72","error":"ExternalModel \"e2e-legacy-migration-d1dc448d\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-d1dc448d\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.7499633,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"2cd7b505-65b2-4ac7-8d43-c3ca605c1b8f"} {"level":"info","ts":1789147890.7499843,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"2cd7b505-65b2-4ac7-8d43-c3ca605c1b8f","provider":"e2e-legacy-migration-d1dc448d","phase":"Failed"} {"level":"info","ts":1789147890.751932,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"04ce1a17-69c0-4bd3-8c54-299c444f74d0","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.756851,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"b7662b03-78db-48bf-8134-a33e17420906","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.7568812,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"b7662b03-78db-48bf-8134-a33e17420906","modelName":"e2e-legacy-migration-d1dc448d","resolvedRefs":1} {"level":"info","ts":1789147890.7601447,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"04ce1a17-69c0-4bd3-8c54-299c444f74d0","name":"e2e-legacy-migration-d1dc448d"} {"level":"error","ts":1789147890.760733,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"2cd7b505-65b2-4ac7-8d43-c3ca605c1b8f","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-d1dc448d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789147890.7607818,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"2cd7b505-65b2-4ac7-8d43-c3ca605c1b8f","error":"ExternalModel \"e2e-legacy-migration-d1dc448d\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-d1dc448d\" is not ready (phase: Failed)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.7608435,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"acdafee1-b289-4b69-bea5-f0ae83c21dad"} {"level":"info","ts":1789147890.7608807,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"acdafee1-b289-4b69-bea5-f0ae83c21dad","provider":"e2e-legacy-migration-d1dc448d","phase":"Failed"} {"level":"info","ts":1789147890.7703314,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"04ce1a17-69c0-4bd3-8c54-299c444f74d0","provider":"openai","model":"e2e-legacy-migration-d1dc448d","targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789147890.7703652,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"acdafee1-b289-4b69-bea5-f0ae83c21dad","error":"ExternalModel \"e2e-legacy-migration-d1dc448d\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-d1dc448d\" is not ready (phase: Failed)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.771781,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8ec65326-da7b-4f23-87f2-d2719b172fdd","service":"e2e-legacy-migration-d1dc448d","serviceEntry":"e2e-legacy-migration-d1dc448d","destinationRule":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147890.771898,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"e228f76c-62f7-420e-81ec-dfd30692ba32","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.7719476,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"e228f76c-62f7-420e-81ec-dfd30692ba32","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789147890.7720873,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"c67ecfbc-5526-409f-8cd5-b870f9c52a6d","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.772178,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"c67ecfbc-5526-409f-8cd5-b870f9c52a6d","modelName":"e2e-legacy-migration-d1dc448d","resolvedRefs":1} {"level":"info","ts":1789147890.772092,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"57cf5671-a4b3-466f-9490-21129cb9975e"} {"level":"info","ts":1789147890.772234,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"57cf5671-a4b3-466f-9490-21129cb9975e","name":"e2e-legacy-migration-d1dc448d"} {"level":"error","ts":1789147890.7765257,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8ec65326-da7b-4f23-87f2-d2719b172fdd","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-d1dc448d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:97\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.776596,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"81709f97-6200-4f25-bd12-97b2feb62a91"} {"level":"info","ts":1789147890.7766342,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"81709f97-6200-4f25-bd12-97b2feb62a91","name":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147890.791861,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"97766c15-2ff0-41ab-b1bc-10685e902886","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.7919493,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"97766c15-2ff0-41ab-b1bc-10685e902886","modelName":"e2e-legacy-migration-d1dc448d","resolvedRefs":1} {"level":"error","ts":1789147890.8041546,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"57cf5671-a4b3-466f-9490-21129cb9975e","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-d1dc448d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789147890.8042026,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"57cf5671-a4b3-466f-9490-21129cb9975e","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-d1dc448d\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.804261,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"a6067512-277d-4e16-93e3-8caf1be284b5"} {"level":"info","ts":1789147890.8043149,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"a6067512-277d-4e16-93e3-8caf1be284b5","name":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147890.8105319,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"81709f97-6200-4f25-bd12-97b2feb62a91","service":"e2e-legacy-migration-d1dc448d","serviceEntry":"e2e-legacy-migration-d1dc448d","destinationRule":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147890.81624,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"0c4b723d-b294-45c2-9b46-e754571cd729","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.816353,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"0c4b723d-b294-45c2-9b46-e754571cd729","modelName":"e2e-legacy-migration-d1dc448d","resolvedRefs":1} {"level":"info","ts":1789147890.8201156,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"a6067512-277d-4e16-93e3-8caf1be284b5","httpRoute":"e2e-legacy-migration-d1dc448d","providers":["e2e-legacy-migration-d1dc448d"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789147890.8255198,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"a6067512-277d-4e16-93e3-8caf1be284b5","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-d1dc448d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.8256257,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"967dfee3-7c5f-4599-a265-546e57b5555f"} {"level":"info","ts":1789147890.8256934,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"967dfee3-7c5f-4599-a265-546e57b5555f","name":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147890.8417196,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"967dfee3-7c5f-4599-a265-546e57b5555f","httpRoute":"e2e-legacy-migration-d1dc448d","providers":["e2e-legacy-migration-d1dc448d"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789147890.8424113,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"e7c317fe-abb8-4296-bb06-e24cba1831fc","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147890.8424888,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"e7c317fe-abb8-4296-bb06-e24cba1831fc","modelName":"e2e-legacy-migration-d1dc448d","resolvedRefs":1} {"level":"error","ts":1789147890.8492599,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"967dfee3-7c5f-4599-a265-546e57b5555f","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-d1dc448d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789147890.8493533,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"5c747744-40b8-4c04-9967-186bd6463efe"} {"level":"info","ts":1789147890.8494208,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"5c747744-40b8-4c04-9967-186bd6463efe","name":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147890.8616574,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"5c747744-40b8-4c04-9967-186bd6463efe","httpRoute":"e2e-legacy-migration-d1dc448d","providers":["e2e-legacy-migration-d1dc448d"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789147891.0022526,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8429a793-9513-46da-9396-a30b08e8f163","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147891.005349,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8429a793-9513-46da-9396-a30b08e8f163","name":"e2e-legacy-migration-d1dc448d"} {"level":"info","ts":1789147891.0101871,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"8429a793-9513-46da-9396-a30b08e8f163","provider":"openai","model":"e2e-legacy-migration-d1dc448d","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789147893.6140234,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"958c03f1-d8d5-4d68-a99b-a99592a3310c","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147893.6199732,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"3cc67972-4236-448d-89a6-793fb162a5c8","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147893.6200287,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"3cc67972-4236-448d-89a6-793fb162a5c8","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147893.6200624,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"dcbc7e56-a3b2-4280-8585-e6352a0457ee"} {"level":"info","ts":1789147893.6211648,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"24a423ea-30e5-4ffc-bdfd-1da35bb0eb11","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147893.6211855,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"24a423ea-30e5-4ffc-bdfd-1da35bb0eb11","name":"e2e-legacy-migration-d1dc448d","namespace":"llm"} {"level":"info","ts":1789147893.6212342,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-d1dc448d","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-d1dc448d","reconcileID":"6db56068-2046-442c-a276-06aa1c19affa"} {"level":"info","ts":1789147894.0184195,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147894.0185325,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.018549,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.0185626,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","EoS":true} {"level":"info","ts":1789147894.0185878,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","header":"x-maas-subscription"} {"level":"info","ts":1789147894.0186012,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","header":"x-maas-group"} {"level":"info","ts":1789147894.0186067,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","header":"x-maas-username"} {"level":"info","ts":1789147894.0186214,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789147894.0186315,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","modelName":"e2e-external-model"} {"level":"info","ts":1789147894.0186431,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789147894.0186665,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789147894.0186846,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","authType":"apikey"} {"level":"info","ts":1789147894.0187118,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.0681422,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.0681744,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.0681987,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","EoS":false} {"level":"info","ts":1789147894.0682151,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.0682411,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b","EoS":true} {"level":"info","ts":1789147894.0682483,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c0e483158c0f78355a2eb5fbcf19d62f","span_id":"0a2144b5bd1d4df8","x-request-id":"d6d7ee06-c9e7-4f7d-a5f1-517cfbfc509b"} {"level":"info","ts":1789147894.1095505,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147894.109674,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.1096902,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.1097038,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","EoS":true} {"level":"info","ts":1789147894.1097288,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","header":"x-maas-username"} {"level":"info","ts":1789147894.1097383,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","header":"x-maas-subscription"} {"level":"info","ts":1789147894.1097436,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","header":"x-maas-group"} {"level":"info","ts":1789147894.1097596,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789147894.109794,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.2571354,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.2571712,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.2571945,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","EoS":false} {"level":"info","ts":1789147894.257213,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.2572477,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c","EoS":true} {"level":"info","ts":1789147894.2572567,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"175b45cbc0026b3142d5492fe61072cc","span_id":"d4933367152c20a0","x-request-id":"b66d5a51-28a5-4c09-a4bf-cb686a10bb2c"} {"level":"info","ts":1789147894.3027678,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147894.3028886,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"info","ts":1789147894.3029056,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"info","ts":1789147894.302919,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658","EoS":true} {"level":"info","ts":1789147894.302944,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658","header":"x-maas-group"} {"level":"info","ts":1789147894.3029537,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658","header":"x-maas-username"} {"level":"info","ts":1789147894.302959,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658","header":"x-maas-subscription"} {"level":"info","ts":1789147894.3030193,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"info","ts":1789147894.3581378,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"info","ts":1789147894.3581746,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"info","ts":1789147894.3581996,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658","EoS":false} {"level":"info","ts":1789147894.3582184,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"info","ts":1789147894.3582513,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658","EoS":true} {"level":"info","ts":1789147894.3582628,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dde7d899c198b764f0299650f67b4e7c","span_id":"d8ec690a0cb9c401","x-request-id":"98825dd8-5d2b-474b-bff0-c8947aa85658"} {"level":"error","ts":1789147894.9255686,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789147895.261127,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"80bd314a-fc81-4e62-971c-7fe18583f552","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789147895.2611895,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"80bd314a-fc81-4e62-971c-7fe18583f552","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789147895.2612495,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"47d8c970-3b0e-4da9-b961-ecf48ffe01d3"} {"level":"info","ts":1789147895.3989,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"fdbb2e7d-35ad-49e6-9355-35b8522f97b7"} {"level":"info","ts":1789147895.3989758,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"9b7c35f6-8f6f-4d98-93e2-baded2691e7f","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789147895.399031,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"9b7c35f6-8f6f-4d98-93e2-baded2691e7f","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789147895.5094764,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"ee49b7c1-de53-455a-b844-97a890bd70f2","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789147895.5095387,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"ee49b7c1-de53-455a-b844-97a890bd70f2","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789147904.9263089,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789147914.9268832,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789147915.2757146,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147915.275834,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.275852,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.2758675,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","EoS":true} {"level":"info","ts":1789147915.2758992,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","header":"x-maas-group"} {"level":"info","ts":1789147915.2759094,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","header":"x-maas-subscription"} {"level":"info","ts":1789147915.2759142,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","header":"x-maas-username"} {"level":"info","ts":1789147915.275928,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789147915.275955,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.2769732,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.2770562,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.277081,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","EoS":false} {"level":"info","ts":1789147915.2770932,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.2771204,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75","EoS":true} {"level":"info","ts":1789147915.27713,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8512bfbcd4da541f59b73e67338b130f","span_id":"14b7040489522050","x-request-id":"fe8c329d-5992-454c-9948-0a1ad5be9b75"} {"level":"info","ts":1789147915.3107457,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147915.3108325,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3108437,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3108547,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","EoS":true} {"level":"info","ts":1789147915.310876,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","header":"x-maas-subscription"} {"level":"info","ts":1789147915.310886,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","header":"x-maas-username"} {"level":"info","ts":1789147915.310891,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","header":"x-maas-group"} {"level":"info","ts":1789147915.310906,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789147915.310931,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3118336,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3118575,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3118765,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","EoS":false} {"level":"info","ts":1789147915.3118954,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3119276,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc","EoS":true} {"level":"info","ts":1789147915.3119369,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4bee978ced42e8f9e08627828a19763a","span_id":"4cedc9faf1abef92","x-request-id":"ea6ccb5c-af38-4f33-b911-9d2b220051cc"} {"level":"info","ts":1789147915.3792787,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147915.379461,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.3795109,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.3795526,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","EoS":true} {"level":"info","ts":1789147915.3796065,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","header":"x-maas-subscription"} {"level":"info","ts":1789147915.3796396,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","header":"x-maas-group"} {"level":"info","ts":1789147915.3796659,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","header":"x-maas-username"} {"level":"info","ts":1789147915.3797066,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789147915.3797662,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.3809237,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.380949,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.38097,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","EoS":false} {"level":"info","ts":1789147915.381073,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.381127,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae","EoS":true} {"level":"info","ts":1789147915.3811429,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f295e5a6768435ebea9150589582ecbe","span_id":"f8116500b861d17b","x-request-id":"e02b9cb3-6004-43af-9f1f-9d4bdaa1eaae"} {"level":"info","ts":1789147915.4167914,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147915.4168925,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147915.4169068,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147915.4169188,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","EoS":true} {"level":"info","ts":1789147915.41695,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","header":"x-maas-group"} {"level":"info","ts":1789147915.4169602,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","header":"x-maas-subscription"} {"level":"info","ts":1789147915.416966,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","header":"x-maas-username"} {"level":"info","ts":1789147915.4169834,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789147915.417024,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147915.4178553,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147915.4178753,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147915.4178925,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","EoS":false} {"level":"info","ts":1789147915.4179075,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147915.417933,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4","EoS":true} {"level":"info","ts":1789147915.4179425,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0a76f9cee54953076c9e394df773b5cd","span_id":"b4a981255630e361","x-request-id":"190f8696-e619-4722-a178-fb983cf183e4"} {"level":"info","ts":1789147928.096961,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147928.0970821,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"790528aa5dde1077f099181e17d1b64f","span_id":"f4136c3216c2b76a","x-request-id":"3120c59c-83d5-410d-9cf9-0838bd2476f9"} {"level":"info","ts":1789147928.0970953,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"790528aa5dde1077f099181e17d1b64f","span_id":"f4136c3216c2b76a","x-request-id":"3120c59c-83d5-410d-9cf9-0838bd2476f9"} {"level":"info","ts":1789147928.0971074,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"790528aa5dde1077f099181e17d1b64f","span_id":"f4136c3216c2b76a","x-request-id":"3120c59c-83d5-410d-9cf9-0838bd2476f9","EoS":true} {"level":"info","ts":1789147928.097158,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"790528aa5dde1077f099181e17d1b64f","span_id":"f4136c3216c2b76a","x-request-id":"3120c59c-83d5-410d-9cf9-0838bd2476f9"} {"level":"info","ts":1789147928.2396853,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147928.2397878,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ea2a215f46bacb84627140593e7b38ba","span_id":"d6591cf34b97c79f","x-request-id":"4df1fc2a-5a7c-4377-b4af-701103a25fe7"} {"level":"info","ts":1789147928.2398002,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ea2a215f46bacb84627140593e7b38ba","span_id":"d6591cf34b97c79f","x-request-id":"4df1fc2a-5a7c-4377-b4af-701103a25fe7"} {"level":"info","ts":1789147928.2398126,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ea2a215f46bacb84627140593e7b38ba","span_id":"d6591cf34b97c79f","x-request-id":"4df1fc2a-5a7c-4377-b4af-701103a25fe7","EoS":true} {"level":"info","ts":1789147928.2398558,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ea2a215f46bacb84627140593e7b38ba","span_id":"d6591cf34b97c79f","x-request-id":"4df1fc2a-5a7c-4377-b4af-701103a25fe7"} {"level":"info","ts":1789147928.3899846,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789147928.3900979,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1ec69c91276154c354db4bdf75906317","span_id":"cf3238597b509296","x-request-id":"43a69c86-ddf9-4026-a2a2-43ba1fd0b131"} {"level":"info","ts":1789147928.3901103,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1ec69c91276154c354db4bdf75906317","span_id":"cf3238597b509296","x-request-id":"43a69c86-ddf9-4026-a2a2-43ba1fd0b131"} {"level":"info","ts":1789147928.3901215,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1ec69c91276154c354db4bdf75906317","span_id":"cf3238597b509296","x-request-id":"43a69c86-ddf9-4026-a2a2-43ba1fd0b131","EoS":true} {"level":"info","ts":1789147928.3901687,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1ec69c91276154c354db4bdf75906317","span_id":"cf3238597b509296","x-request-id":"43a69c86-ddf9-4026-a2a2-43ba1fd0b131"} {"level":"error","ts":1789147929.9288003,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789147944.9300885,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149070.7079594,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149070.7080839,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.708099,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.7081108,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","EoS":true} {"level":"info","ts":1789149070.7081382,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","header":"x-maas-group"} {"level":"info","ts":1789149070.7081485,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","header":"x-maas-subscription"} {"level":"info","ts":1789149070.7081535,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","header":"x-maas-username"} {"level":"info","ts":1789149070.7081695,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789149070.7081957,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.7092261,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.709248,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.7092643,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","EoS":false} {"level":"info","ts":1789149070.709277,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.7093117,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf","EoS":true} {"level":"info","ts":1789149070.709322,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b21e87a35bdad030e89430af27b9d286","span_id":"03ef4496b0596709","x-request-id":"ec7ff348-70cb-4719-b803-a19176efb5bf"} {"level":"info","ts":1789149070.7453508,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149070.745449,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"info","ts":1789149070.7454607,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"info","ts":1789149070.745472,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","EoS":true} {"level":"info","ts":1789149070.745499,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","header":"x-maas-username"} {"level":"info","ts":1789149070.7455096,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","header":"x-maas-subscription"} {"level":"info","ts":1789149070.745515,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","header":"x-maas-group"} {"level":"info","ts":1789149070.7455306,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789149070.7455587,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"info","ts":1789149070.7464616,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"info","ts":1789149070.746487,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"info","ts":1789149070.7465048,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","EoS":false} {"level":"info","ts":1789149070.746521,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"info","ts":1789149070.7465613,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f","EoS":true} {"level":"info","ts":1789149070.7465775,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1069943109ee1813ec2bf8df30677915","span_id":"76ee6017a239ea5a","x-request-id":"e7c68ce0-8166-4eba-a67f-b0e38e70d62f"} {"level":"error","ts":1789149085.0671623,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149402.8938637,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149402.893964,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149402.8939776,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149402.8940022,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","EoS":true} {"level":"info","ts":1789149402.894029,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","header":"x-maas-group"} {"level":"info","ts":1789149402.8940394,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","header":"x-maas-username"} {"level":"info","ts":1789149402.894044,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","header":"x-maas-subscription"} {"level":"info","ts":1789149402.8940587,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149402.8940833,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149402.906345,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149402.9063683,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149402.9063838,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","EoS":false} {"level":"info","ts":1789149402.9063964,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149402.9064229,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff","EoS":true} {"level":"info","ts":1789149402.9064298,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8effb7200fb5cb08737849edb5d7cab8","span_id":"e1a2c26783b36308","x-request-id":"b812df09-669d-45ad-8b71-32d6baff24ff"} {"level":"info","ts":1789149403.055714,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149403.0558233,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.055836,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.0558476,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","EoS":true} {"level":"info","ts":1789149403.0558696,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","header":"x-maas-username"} {"level":"info","ts":1789149403.0558803,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","header":"x-maas-subscription"} {"level":"info","ts":1789149403.0558846,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","header":"x-maas-group"} {"level":"info","ts":1789149403.0558996,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149403.055923,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.0575504,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.0575712,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.0575862,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","EoS":false} {"level":"info","ts":1789149403.0575979,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.057618,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543","EoS":true} {"level":"info","ts":1789149403.0576272,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4c85e35e218c4a1bf0a385e916b84d91","span_id":"84a57561dead7d4b","x-request-id":"0e5fc6b4-407b-4949-b721-d1b74f646543"} {"level":"info","ts":1789149403.1950164,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149403.1951277,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.1951427,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.1951556,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","EoS":true} {"level":"info","ts":1789149403.1951814,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","header":"x-maas-username"} {"level":"info","ts":1789149403.19519,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","header":"x-maas-group"} {"level":"info","ts":1789149403.1951954,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","header":"x-maas-subscription"} {"level":"info","ts":1789149403.195212,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149403.1952379,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.1958497,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.195868,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.1958792,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","EoS":false} {"level":"info","ts":1789149403.1958907,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.1959102,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a","EoS":true} {"level":"info","ts":1789149403.1959195,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0ab3fb328a5e92ed9082bff1d01958d1","span_id":"8f25c9ce76c94e47","x-request-id":"aec9d86b-2700-4505-8e74-bc6c08492a0a"} {"level":"info","ts":1789149403.3307235,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149403.3308265,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.3308394,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.3308508,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","EoS":true} {"level":"info","ts":1789149403.330874,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","header":"x-maas-group"} {"level":"info","ts":1789149403.3308845,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","header":"x-maas-subscription"} {"level":"info","ts":1789149403.3308895,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","header":"x-maas-username"} {"level":"info","ts":1789149403.3309047,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149403.3309324,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.3316503,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.3316748,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.3316872,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","EoS":false} {"level":"info","ts":1789149403.3317037,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.3317292,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe","EoS":true} {"level":"info","ts":1789149403.3317363,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"51e3072df3165ec83a46cdca1825ec94","span_id":"b7df2e09fe6261f4","x-request-id":"32021aab-771e-463f-87cc-57e34f380efe"} {"level":"info","ts":1789149403.4698932,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149403.470022,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149403.4700372,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149403.470051,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","EoS":true} {"level":"info","ts":1789149403.4700751,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","header":"x-maas-group"} {"level":"info","ts":1789149403.4700885,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","header":"x-maas-username"} {"level":"info","ts":1789149403.4700968,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","header":"x-maas-subscription"} {"level":"info","ts":1789149403.4701142,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149403.4701395,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149403.4708173,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149403.4708376,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149403.4708529,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","EoS":false} {"level":"info","ts":1789149403.4708662,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149403.4708865,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454","EoS":true} {"level":"info","ts":1789149403.4708943,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"75da36e73a8c3b64964e0604c8d60e3c","span_id":"c0c8382f6b121e88","x-request-id":"5a97d775-bcf7-4243-a5a9-26cd6cfbc454"} {"level":"info","ts":1789149412.2902834,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149412.2903855,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.2903976,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.2904093,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","EoS":true} {"level":"info","ts":1789149412.2904336,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","header":"x-maas-subscription"} {"level":"info","ts":1789149412.2904437,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","header":"x-maas-group"} {"level":"info","ts":1789149412.2904496,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","header":"x-maas-username"} {"level":"info","ts":1789149412.2904649,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149412.2904897,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.2912652,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.291287,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.2913,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","EoS":false} {"level":"info","ts":1789149412.2913108,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.2913342,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9","EoS":true} {"level":"info","ts":1789149412.2913415,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e9403e0e98e2cfd9684c8b2a322a3222","span_id":"c33216717767b141","x-request-id":"a47d82ca-7d13-497c-930a-00c3ae2559f9"} {"level":"info","ts":1789149412.3949175,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149412.3950322,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758"} {"level":"info","ts":1789149412.3956625,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758"} {"level":"info","ts":1789149412.3956826,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758"} {"level":"info","ts":1789149412.3956969,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758","EoS":false} {"level":"info","ts":1789149412.3957086,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758"} {"level":"info","ts":1789149412.395734,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758","EoS":true} {"level":"info","ts":1789149412.3957431,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bab64d2979887c34f366162bacb1f831","span_id":"812fca3c297548ca","x-request-id":"9273c23f-1829-4814-9dc0-1e2de2093758"} {"level":"info","ts":1789149413.1549795,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149413.1551335,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149413.1551473,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149413.1551592,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","EoS":true} {"level":"info","ts":1789149413.155183,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","header":"x-maas-username"} {"level":"info","ts":1789149413.1551945,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","header":"x-maas-subscription"} {"level":"info","ts":1789149413.155201,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","header":"x-maas-group"} {"level":"info","ts":1789149413.1552165,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149413.1552424,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149413.156161,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149413.1561868,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149413.1562033,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","EoS":false} {"level":"info","ts":1789149413.1562204,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149413.1562505,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697","EoS":true} {"level":"info","ts":1789149413.1562603,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e76d8bf49f4b61b3f2df286d79549518","span_id":"b413d7981996fbc0","x-request-id":"38354636-c317-4ee5-8ec4-649964578697"} {"level":"info","ts":1789149414.0715947,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149414.0716934,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.071706,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.0717177,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","EoS":true} {"level":"info","ts":1789149414.0717409,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","header":"x-maas-subscription"} {"level":"info","ts":1789149414.0717514,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","header":"x-maas-group"} {"level":"info","ts":1789149414.0717576,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","header":"x-maas-username"} {"level":"info","ts":1789149414.0717726,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149414.0717995,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.0726552,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.0726743,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.072689,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","EoS":false} {"level":"info","ts":1789149414.0727038,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.0727289,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4","EoS":true} {"level":"info","ts":1789149414.0727377,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"78bf82a6d8f418f0a6a9a975fad9834a","span_id":"da7ca095ef8f23e3","x-request-id":"1ddab868-4b52-4948-ab1a-3a525a3f68b4"} {"level":"info","ts":1789149414.784091,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149414.7841902,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"info","ts":1789149414.7842023,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"info","ts":1789149414.7842135,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","EoS":true} {"level":"info","ts":1789149414.784237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","header":"x-maas-username"} {"level":"info","ts":1789149414.7842464,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","header":"x-maas-group"} {"level":"info","ts":1789149414.7842536,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","header":"x-maas-subscription"} {"level":"info","ts":1789149414.7842708,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149414.784297,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"info","ts":1789149414.7851956,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"info","ts":1789149414.7852154,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"info","ts":1789149414.7852376,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","EoS":false} {"level":"info","ts":1789149414.7852569,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"info","ts":1789149414.7852917,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57","EoS":true} {"level":"info","ts":1789149414.7853026,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5ab5ff73ad9bdd29d769e39ac6c225da","span_id":"312ce3b857519e33","x-request-id":"de23012b-59f7-4e5f-9a73-972b37bccc57"} {"level":"error","ts":1789149415.1044035,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149415.8310885,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149415.8311875,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"info","ts":1789149415.8312,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"info","ts":1789149415.8312116,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","EoS":true} {"level":"info","ts":1789149415.831234,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","header":"x-maas-subscription"} {"level":"info","ts":1789149415.8312447,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","header":"x-maas-username"} {"level":"info","ts":1789149415.8312497,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","header":"x-maas-group"} {"level":"info","ts":1789149415.8312654,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149415.831295,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"info","ts":1789149415.8321464,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"info","ts":1789149415.8321674,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"info","ts":1789149415.832182,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","EoS":false} {"level":"info","ts":1789149415.8321953,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"info","ts":1789149415.8322227,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173","EoS":true} {"level":"info","ts":1789149415.8322315,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad51c1f6d0920384f897864273b299f0","span_id":"9ee342f8b9a3f99c","x-request-id":"f457a3a5-e447-4029-949c-5340b716f173"} {"level":"error","ts":1789149425.1050506,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149431.13182,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149431.1319287,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"info","ts":1789149431.1319427,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"info","ts":1789149431.1319542,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","EoS":true} {"level":"info","ts":1789149431.1319795,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","header":"x-maas-username"} {"level":"info","ts":1789149431.132014,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","header":"x-maas-subscription"} {"level":"info","ts":1789149431.132023,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","header":"x-maas-group"} {"level":"info","ts":1789149431.1320393,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149431.1320667,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"info","ts":1789149431.132959,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"info","ts":1789149431.1329794,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"info","ts":1789149431.1330109,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","EoS":false} {"level":"info","ts":1789149431.1330268,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"info","ts":1789149431.133047,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1","EoS":true} {"level":"info","ts":1789149431.1330562,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"85594f4b107180cd189fc9c4f6c67805","span_id":"486e5bc32d76c6c6","x-request-id":"2ef478bc-fdb0-41ff-9113-1f4c3ae40af1"} {"level":"error","ts":1789149435.106096,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789149445.1070483,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149520.7161708,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149520.7163491,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"info","ts":1789149520.7163732,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"info","ts":1789149520.7163908,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","EoS":true} {"level":"info","ts":1789149520.7164202,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","header":"x-maas-group"} {"level":"info","ts":1789149520.716431,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","header":"x-maas-username"} {"level":"info","ts":1789149520.7164364,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","header":"x-maas-subscription"} {"level":"info","ts":1789149520.7164533,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789149520.7164836,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"info","ts":1789149520.7293231,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"info","ts":1789149520.7293496,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"info","ts":1789149520.7293668,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","EoS":false} {"level":"info","ts":1789149520.7293787,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"info","ts":1789149520.7294095,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979","EoS":true} {"level":"info","ts":1789149520.7294195,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"da7ded7bf24c076a501077f1b3520611","span_id":"e817243203cfb4df","x-request-id":"ccaed415-1b54-482f-b9d8-a28806fd7979"} {"level":"error","ts":1789149535.1165307,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149609.4947534,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149609.4948647,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"info","ts":1789149609.4948788,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"info","ts":1789149609.4948912,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","EoS":true} {"level":"info","ts":1789149609.4949164,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","header":"x-maas-group"} {"level":"info","ts":1789149609.4949257,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","header":"x-maas-subscription"} {"level":"info","ts":1789149609.494931,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","header":"x-maas-username"} {"level":"info","ts":1789149609.494946,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789149609.4949741,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"info","ts":1789149609.4957018,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"info","ts":1789149609.495721,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"info","ts":1789149609.4957333,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","EoS":false} {"level":"info","ts":1789149609.4957445,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"info","ts":1789149609.4957685,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901","EoS":true} {"level":"info","ts":1789149609.4957783,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d6c8deff681dc507660df10a879d356a","span_id":"f265ccc89a7ac6cb","x-request-id":"e1af73ac-a172-4e37-8975-50750bee8901"} {"level":"error","ts":1789149620.1269433,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149623.1794875,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149623.1796012,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1796172,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1796362,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","EoS":true} {"level":"info","ts":1789149623.1796703,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","header":"x-maas-username"} {"level":"info","ts":1789149623.1796823,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","header":"x-maas-group"} {"level":"info","ts":1789149623.1796925,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","header":"x-maas-subscription"} {"level":"info","ts":1789149623.1797159,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789149623.1797526,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1932392,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1933086,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1933308,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","EoS":false} {"level":"info","ts":1789149623.1933577,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1934805,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","EoS":false} {"level":"info","ts":1789149623.193515,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.1935515,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224","EoS":true} {"level":"info","ts":1789149623.193567,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"09efe815122812c8b0ffb25289d03d83","span_id":"7f0f6934592ea72f","x-request-id":"749db598-38b2-4abd-b66a-f43c4cb89224"} {"level":"info","ts":1789149623.3317716,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149623.3318853,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.3318996,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.3319178,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","EoS":true} {"level":"info","ts":1789149623.3319576,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","header":"x-maas-username"} {"level":"info","ts":1789149623.331978,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","header":"x-maas-subscription"} {"level":"info","ts":1789149623.3319843,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","header":"x-maas-group"} {"level":"info","ts":1789149623.3320215,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789149623.3320591,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.3329687,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.333002,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.3330243,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","EoS":false} {"level":"info","ts":1789149623.3330379,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.3330648,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","EoS":false} {"level":"info","ts":1789149623.3330772,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.333096,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6","EoS":true} {"level":"info","ts":1789149623.333105,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f02e05dc30de10a6fa4264d93c8e70f","span_id":"9feab4aaa57e9369","x-request-id":"d43e3725-8146-464a-a4a2-f2c8a92f4bd6"} {"level":"info","ts":1789149623.4674299,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149623.4675477,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4675632,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4675765,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","EoS":true} {"level":"info","ts":1789149623.4676018,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","header":"x-maas-group"} {"level":"info","ts":1789149623.467613,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","header":"x-maas-username"} {"level":"info","ts":1789149623.4676185,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","header":"x-maas-subscription"} {"level":"info","ts":1789149623.467635,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789149623.467662,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4700217,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4700441,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4700618,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","EoS":false} {"level":"info","ts":1789149623.470074,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4700968,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","EoS":false} {"level":"info","ts":1789149623.470112,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"info","ts":1789149623.4701302,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666","EoS":true} {"level":"info","ts":1789149623.4701414,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b474570fc51a5047ee28cc004fe6d24f","span_id":"6a14e4a8325fdcff","x-request-id":"244367db-294d-4703-bd63-3edddaab2666"} {"level":"error","ts":1789149635.127648,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789149637.3499262,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789149637.3500674,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"info","ts":1789149637.3500862,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"info","ts":1789149637.3501024,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","EoS":true} {"level":"info","ts":1789149637.3501296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","header":"x-maas-subscription"} {"level":"info","ts":1789149637.350137,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","header":"x-maas-username"} {"level":"info","ts":1789149637.3501422,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","header":"x-maas-group"} {"level":"info","ts":1789149637.3501596,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789149637.3501906,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"info","ts":1789149637.3524995,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"info","ts":1789149637.3525226,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"info","ts":1789149637.3525412,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","EoS":false} {"level":"info","ts":1789149637.3525534,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"info","ts":1789149637.3525834,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673","EoS":true} {"level":"info","ts":1789149637.3525941,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"481d9318abee66dad33bd7b8d9571cbb","span_id":"ba02712f44048e87","x-request-id":"06c94e51-18c1-4ca6-8008-4672083f6673"} {"level":"error","ts":1789149650.1297088,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}