{"level":"info","ts":1789475921.4919457,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b","header":"x-maas-username"} {"level":"info","ts":1789475921.4919515,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b","header":"x-maas-group"} {"level":"info","ts":1789475921.4919648,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789475921.4919949,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b"} {"level":"info","ts":1789475921.4927883,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b"} {"level":"info","ts":1789475921.4928062,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b"} {"level":"info","ts":1789475921.4928217,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b","EoS":false} {"level":"info","ts":1789475921.4928384,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b"} {"level":"info","ts":1789475921.492864,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b","EoS":true} {"level":"info","ts":1789475921.4928734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e09a443db09613a4ed9ce4aa3c1a05fe","span_id":"ea04287f649342be","x-request-id":"60b979b8-9c76-4401-b12f-cd8078c88a6b"} {"level":"info","ts":1789475922.3152537,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"ec74db76-c3e8-4104-b09b-d7340d73dddc","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789475922.315477,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"ec74db76-c3e8-4104-b09b-d7340d73dddc","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789475922.3154342,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"6c77ac47-43d7-4c07-b2b6-fa176328078b"} {"level":"info","ts":1789475926.610418,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475926.610536,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0c8dd8252eeed0f2993df376da64e66a","span_id":"90bdad1b3226703c","x-request-id":"dae7d25e-fc0d-47f9-997b-521c3086865d"} {"level":"info","ts":1789475926.6105537,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0c8dd8252eeed0f2993df376da64e66a","span_id":"90bdad1b3226703c","x-request-id":"dae7d25e-fc0d-47f9-997b-521c3086865d"} {"level":"info","ts":1789475926.6105728,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0c8dd8252eeed0f2993df376da64e66a","span_id":"90bdad1b3226703c","x-request-id":"dae7d25e-fc0d-47f9-997b-521c3086865d","EoS":true} {"level":"info","ts":1789475926.6106224,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0c8dd8252eeed0f2993df376da64e66a","span_id":"90bdad1b3226703c","x-request-id":"dae7d25e-fc0d-47f9-997b-521c3086865d","path":"/llm/nonexistent-model-xyz/v1/chat/completions"} {"level":"info","ts":1789475926.6106603,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0c8dd8252eeed0f2993df376da64e66a","span_id":"90bdad1b3226703c","x-request-id":"dae7d25e-fc0d-47f9-997b-521c3086865d"} {"level":"info","ts":1789475926.9231045,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"b93a0f64-ab48-44f5-a0f1-1e485675b5b7","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9268095,"caller":"legacymigration/reconciler.go:147","msg":"creating ExternalProvider","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"b93a0f64-ab48-44f5-a0f1-1e485675b5b7","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475926.9432776,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"214d06d9-7ddd-4d61-b231-bcb867596bbb","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9433227,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"214d06d9-7ddd-4d61-b231-bcb867596bbb","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789475926.9433966,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"27dd22ed-2850-4e13-9286-46a45aa4a1f7"} {"level":"error","ts":1789475926.947011,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"b93a0f64-ab48-44f5-a0f1-1e485675b5b7","error":"failed to apply ExternalProvider: externalproviders.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.9491265,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"35645adc-9636-455c-9ace-66cde24e561a","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.949161,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"35645adc-9636-455c-9ace-66cde24e561a","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"info","ts":1789475926.9492195,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"fdbe8d28-9da8-4ec4-8643-0ae842a12ba3"} {"level":"info","ts":1789475926.9542575,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"a69c98f7-390a-43cd-be1c-eb8f77fca32a","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9543078,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"a69c98f7-390a-43cd-be1c-eb8f77fca32a","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789475926.9545515,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"6e844ec0-fae3-459c-a905-de6111c333dc","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.954599,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"6e844ec0-fae3-459c-a905-de6111c333dc","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"error","ts":1789475926.9547222,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"27dd22ed-2850-4e13-9286-46a45aa4a1f7","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:84\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.9547791,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"27dd22ed-2850-4e13-9286-46a45aa4a1f7","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475926.9555445,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"cacd327e-710b-40b2-955b-9a733d72d5f4","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.958423,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"be6df231-c99e-4b49-81ec-4ebc0fbd1f95","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9584556,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"be6df231-c99e-4b49-81ec-4ebc0fbd1f95","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"error","ts":1789475926.9590912,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"fdbe8d28-9da8-4ec4-8643-0ae842a12ba3","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.9591362,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"fdbe8d28-9da8-4ec4-8643-0ae842a12ba3","provider":"e2e-legacy-migration-0d1d587c","phase":"Pending"} {"level":"info","ts":1789475926.963023,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"cacd327e-710b-40b2-955b-9a733d72d5f4","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475926.96688,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"185ae4cd-1d05-455c-baaa-24ad9d03fc70","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9669504,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"185ae4cd-1d05-455c-baaa-24ad9d03fc70","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"error","ts":1789475926.9670355,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"fdbe8d28-9da8-4ec4-8643-0ae842a12ba3","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789475926.967121,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"fdbe8d28-9da8-4ec4-8643-0ae842a12ba3","error":"ExternalModel \"e2e-legacy-migration-0d1d587c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-0d1d587c\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.967197,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"09f03af2-d0fc-4d51-b039-cc4a55ecb0ef"} {"level":"info","ts":1789475926.9672403,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"09f03af2-d0fc-4d51-b039-cc4a55ecb0ef","provider":"e2e-legacy-migration-0d1d587c","phase":"Pending"} {"level":"error","ts":1789475926.9682255,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"cacd327e-710b-40b2-955b-9a733d72d5f4","error":"failed to apply ExternalModel: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789475926.972748,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"09f03af2-d0fc-4d51-b039-cc4a55ecb0ef","error":"ExternalModel \"e2e-legacy-migration-0d1d587c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-0d1d587c\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789475926.9731781,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"27dd22ed-2850-4e13-9286-46a45aa4a1f7","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789475926.9732215,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"27dd22ed-2850-4e13-9286-46a45aa4a1f7","error":"failed to apply Service: services \"e2e-legacy-migration-0d1d587c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.9732778,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"634b86ee-42c7-44e0-bd2a-5273a8ad221b"} {"level":"info","ts":1789475926.9733615,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"634b86ee-42c7-44e0-bd2a-5273a8ad221b","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475926.978863,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"67f8aa74-53eb-4ab1-9706-41455b98a0bc","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9830077,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3b53571f-ef54-4452-82e5-eee1caa21e12"} {"level":"info","ts":1789475926.9830413,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3b53571f-ef54-4452-82e5-eee1caa21e12","provider":"e2e-legacy-migration-0d1d587c","phase":"Pending"} {"level":"info","ts":1789475926.9845226,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"67f8aa74-53eb-4ab1-9706-41455b98a0bc","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475926.9902043,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"634b86ee-42c7-44e0-bd2a-5273a8ad221b","service":"e2e-legacy-migration-0d1d587c","serviceEntry":"e2e-legacy-migration-0d1d587c","destinationRule":"e2e-legacy-migration-0d1d587c"} {"level":"error","ts":1789475926.990338,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3b53571f-ef54-4452-82e5-eee1caa21e12","error":"ExternalModel \"e2e-legacy-migration-0d1d587c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-0d1d587c\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.9937265,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"1a78b495-d31b-4f4f-8ff7-14cdb4d13d9b"} {"level":"info","ts":1789475926.993773,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"1a78b495-d31b-4f4f-8ff7-14cdb4d13d9b","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475926.9940596,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"abf90832-4c9a-413a-a9d3-1a7efb27cd29","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9940937,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"abf90832-4c9a-413a-a9d3-1a7efb27cd29","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"info","ts":1789475926.9941506,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"847b0c74-71b9-4883-b778-5d8a21690629","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475926.9941683,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"847b0c74-71b9-4883-b778-5d8a21690629","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789475926.994597,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"67f8aa74-53eb-4ab1-9706-41455b98a0bc","provider":"openai","model":"e2e-legacy-migration-0d1d587c","targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789475926.9960768,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"634b86ee-42c7-44e0-bd2a-5273a8ad221b","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:97\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475926.9961457,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"ce2f2486-9933-412f-8af1-83064ab46c6f"} {"level":"info","ts":1789475926.9961846,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"ce2f2486-9933-412f-8af1-83064ab46c6f","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475927.0139837,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"ce2f2486-9933-412f-8af1-83064ab46c6f","service":"e2e-legacy-migration-0d1d587c","serviceEntry":"e2e-legacy-migration-0d1d587c","destinationRule":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475927.0143023,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"1e513717-321e-4efc-8c17-69d0d6d7901f","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475927.01436,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"1e513717-321e-4efc-8c17-69d0d6d7901f","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"error","ts":1789475927.0211258,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"1a78b495-d31b-4f4f-8ff7-14cdb4d13d9b","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789475927.021201,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"1a78b495-d31b-4f4f-8ff7-14cdb4d13d9b","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-0d1d587c\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475927.0213025,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"0d6012d5-5a5c-4a1b-83d2-cdc4e919867d"} {"level":"info","ts":1789475927.0213637,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"0d6012d5-5a5c-4a1b-83d2-cdc4e919867d","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475927.0312746,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"0d6012d5-5a5c-4a1b-83d2-cdc4e919867d","httpRoute":"e2e-legacy-migration-0d1d587c","providers":["e2e-legacy-migration-0d1d587c"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789475927.0350142,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"c153a888-e0a9-44bd-a2eb-e47b160030b9","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475927.0350518,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"c153a888-e0a9-44bd-a2eb-e47b160030b9","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"error","ts":1789475927.0388408,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"0d6012d5-5a5c-4a1b-83d2-cdc4e919867d","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475927.0389237,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3f150515-445d-4e46-870c-e6787bf8a3c1"} {"level":"info","ts":1789475927.0389903,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3f150515-445d-4e46-870c-e6787bf8a3c1","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475927.0530305,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"a94f6945-f516-44b2-87e1-08bbace8ba70","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475927.0530984,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"a94f6945-f516-44b2-87e1-08bbace8ba70","modelName":"e2e-legacy-migration-0d1d587c","resolvedRefs":1} {"level":"info","ts":1789475927.0538855,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3f150515-445d-4e46-870c-e6787bf8a3c1","httpRoute":"e2e-legacy-migration-0d1d587c","providers":["e2e-legacy-migration-0d1d587c"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789475927.0596254,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"3f150515-445d-4e46-870c-e6787bf8a3c1","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475927.0597265,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"670f9f57-4ece-48ba-8c57-1dad4ce5177b"} {"level":"info","ts":1789475927.0597937,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"670f9f57-4ece-48ba-8c57-1dad4ce5177b","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475927.0698805,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"670f9f57-4ece-48ba-8c57-1dad4ce5177b","httpRoute":"e2e-legacy-migration-0d1d587c","providers":["e2e-legacy-migration-0d1d587c"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789475927.1321278,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"2cf9ce6f-ef2e-483c-9dea-ff49900d3c0f","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475927.1353338,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"2cf9ce6f-ef2e-483c-9dea-ff49900d3c0f","name":"e2e-legacy-migration-0d1d587c"} {"level":"info","ts":1789475927.1418903,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"2cf9ce6f-ef2e-483c-9dea-ff49900d3c0f","provider":"openai","model":"e2e-legacy-migration-0d1d587c","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789475929.8165174,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"994b6b2a-374c-4bb3-9c51-dfc462108257","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475929.8246627,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"7d33a9c9-008e-477d-b13f-16bf810a48f9"} {"level":"info","ts":1789475929.8247366,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5bec2d15-22d2-432b-808b-aeac720eaed6"} {"level":"error","ts":1789475929.8247733,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5bec2d15-22d2-432b-808b-aeac720eaed6","provider":"e2e-legacy-migration-0d1d587c","error":"ExternalProvider.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475929.8248816,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"f23c7e6f-6e15-4d8e-b127-4724e859285c","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475929.8249424,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"f23c7e6f-6e15-4d8e-b127-4724e859285c","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475929.8246653,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5cdfbe2a-ca6e-4e7b-beaa-c510c9378991","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"error","ts":1789475929.8250353,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5cdfbe2a-ca6e-4e7b-beaa-c510c9378991","provider":"e2e-legacy-migration-0d1d587c","error":"ExternalProvider \"e2e-legacy-migration-0d1d587c\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475929.825102,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5cdfbe2a-ca6e-4e7b-beaa-c510c9378991"} {"level":"info","ts":1789475929.8275595,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"38218cf8-de20-4497-a4cb-84e790b53848","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"info","ts":1789475929.8276231,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"38218cf8-de20-4497-a4cb-84e790b53848","name":"e2e-legacy-migration-0d1d587c","namespace":"llm"} {"level":"error","ts":1789475929.830667,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5bec2d15-22d2-432b-808b-aeac720eaed6","error":"externalmodels.inference.opendatahub.io \"e2e-legacy-migration-0d1d587c\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789475929.830741,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"5bec2d15-22d2-432b-808b-aeac720eaed6","error":"ExternalModel \"e2e-legacy-migration-0d1d587c\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-0d1d587c\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789475929.8308084,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-0d1d587c","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-0d1d587c","reconcileID":"d9f6c57a-3a92-4c98-a3bf-d6e76b56ab6b"} {"level":"info","ts":1789475930.2218578,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475930.2220216,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475930.2220414,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475930.2220595,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","EoS":true} {"level":"info","ts":1789475930.222093,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","header":"x-maas-username"} {"level":"info","ts":1789475930.222105,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","header":"x-maas-group"} {"level":"info","ts":1789475930.2221134,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","header":"x-maas-subscription"} {"level":"info","ts":1789475930.2221363,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789475930.2221506,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","modelName":"e2e-external-model"} {"level":"info","ts":1789475930.222169,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789475930.2221994,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789475930.2222278,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","authType":"apikey"} {"level":"info","ts":1789475930.2222643,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475932.5686195,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475932.5686498,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475932.5686672,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","EoS":false} {"level":"info","ts":1789475932.5686817,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475932.5687132,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0","EoS":true} {"level":"info","ts":1789475932.568722,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d98bcfaebb7dac1084b9ede49d660335","span_id":"ae91b54b688d2f51","x-request-id":"1422c8c8-b9b1-4da2-8da5-db46b63b2ca0"} {"level":"info","ts":1789475932.668308,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475932.6684568,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"info","ts":1789475932.6684723,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"info","ts":1789475932.668495,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","EoS":true} {"level":"info","ts":1789475932.6685305,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","header":"x-maas-username"} {"level":"info","ts":1789475932.6685443,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","header":"x-maas-subscription"} {"level":"info","ts":1789475932.6685526,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","header":"x-maas-group"} {"level":"info","ts":1789475932.668575,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789475932.6686118,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"error","ts":1789475934.865112,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789475935.9019024,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475935.9019969,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"28ce9124f4e9850de9a8369cbde33c7a","span_id":"64b71daffffeef46","x-request-id":"3864bb56-e225-4b74-b89d-bbb74e7ef770"} {"level":"info","ts":1789475935.9020083,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"28ce9124f4e9850de9a8369cbde33c7a","span_id":"64b71daffffeef46","x-request-id":"3864bb56-e225-4b74-b89d-bbb74e7ef770"} {"level":"info","ts":1789475935.9020195,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"28ce9124f4e9850de9a8369cbde33c7a","span_id":"64b71daffffeef46","x-request-id":"3864bb56-e225-4b74-b89d-bbb74e7ef770","EoS":true} {"level":"info","ts":1789475935.9020696,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"28ce9124f4e9850de9a8369cbde33c7a","span_id":"64b71daffffeef46","x-request-id":"3864bb56-e225-4b74-b89d-bbb74e7ef770"} {"level":"info","ts":1789475936.001487,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"info","ts":1789475936.0015166,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"info","ts":1789475936.0015342,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","EoS":false} {"level":"info","ts":1789475936.001548,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"info","ts":1789475936.0015836,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa","EoS":true} {"level":"info","ts":1789475936.0015967,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"80414a9d7d3b4a7a69037eeeec5bed52","span_id":"8a6352e9b7ca4249","x-request-id":"c082ceee-6b06-4a37-b3e5-419b95a1e3aa"} {"level":"info","ts":1789475936.0519016,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475936.0519965,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7d04c915dc4f9384276c46bf0795c08a","span_id":"692c5fa3ed14198e","x-request-id":"95921013-e837-4545-8f4d-5877fca4c6ea"} {"level":"info","ts":1789475936.0520132,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7d04c915dc4f9384276c46bf0795c08a","span_id":"692c5fa3ed14198e","x-request-id":"95921013-e837-4545-8f4d-5877fca4c6ea"} {"level":"info","ts":1789475936.052015,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475936.0520248,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7d04c915dc4f9384276c46bf0795c08a","span_id":"692c5fa3ed14198e","x-request-id":"95921013-e837-4545-8f4d-5877fca4c6ea","EoS":true} {"level":"info","ts":1789475936.0521576,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7d04c915dc4f9384276c46bf0795c08a","span_id":"692c5fa3ed14198e","x-request-id":"95921013-e837-4545-8f4d-5877fca4c6ea"} {"level":"info","ts":1789475936.0521057,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475936.0522032,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475936.0522244,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402","EoS":true} {"level":"info","ts":1789475936.0522528,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402","header":"x-maas-group"} {"level":"info","ts":1789475936.0522676,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402","header":"x-maas-subscription"} {"level":"info","ts":1789475936.0522768,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402","header":"x-maas-username"} {"level":"info","ts":1789475936.0522993,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475936.2245896,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789475936.2247002,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5e515c82ecba8d7ad7425d8e57724fab","span_id":"3f93316608dbc089","x-request-id":"5d5e49f1-755d-43f2-8721-70c813d86aff"} {"level":"info","ts":1789475936.224717,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5e515c82ecba8d7ad7425d8e57724fab","span_id":"3f93316608dbc089","x-request-id":"5d5e49f1-755d-43f2-8721-70c813d86aff"} {"level":"info","ts":1789475936.2247298,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5e515c82ecba8d7ad7425d8e57724fab","span_id":"3f93316608dbc089","x-request-id":"5d5e49f1-755d-43f2-8721-70c813d86aff","EoS":true} {"level":"info","ts":1789475936.2247767,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5e515c82ecba8d7ad7425d8e57724fab","span_id":"3f93316608dbc089","x-request-id":"5d5e49f1-755d-43f2-8721-70c813d86aff"} {"level":"info","ts":1789475938.5701842,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475938.5702136,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475938.5702322,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402","EoS":false} {"level":"info","ts":1789475938.570246,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475938.5702746,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402","EoS":true} {"level":"info","ts":1789475938.5702834,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0a867794c48de290671c0cf0efc7943b","span_id":"21c1eb4789d068c6","x-request-id":"3c9736dc-1ef3-447d-8c8d-f63997aa7402"} {"level":"info","ts":1789475939.3784547,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"995c8b0f-159d-4711-9ef6-98e9e625b9b1","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789475939.3785024,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"995c8b0f-159d-4711-9ef6-98e9e625b9b1","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789475939.3785605,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"9e8f8c65-d9d7-493a-8c72-47ac549f16a8"} {"level":"info","ts":1789475939.5099833,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"b78614f6-faf9-46e0-bddb-9a97e7a8953b","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789475939.510099,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"b78614f6-faf9-46e0-bddb-9a97e7a8953b","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789475939.5101886,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"d1d2f265-b946-40ff-b982-287297e01e93"} {"level":"info","ts":1789475939.6390796,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"17374969-9f49-44ed-95a5-c5b38b11ddb3","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789475939.6391299,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"17374969-9f49-44ed-95a5-c5b38b11ddb3","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789475949.8662915,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477110.2367437,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477110.2368584,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2368739,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2368865,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","EoS":true} {"level":"info","ts":1789477110.2369165,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","header":"x-maas-username"} {"level":"info","ts":1789477110.2369263,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","header":"x-maas-group"} {"level":"info","ts":1789477110.2369351,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","header":"x-maas-subscription"} {"level":"info","ts":1789477110.2369545,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789477110.2369823,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2377687,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2377946,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2378104,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","EoS":false} {"level":"info","ts":1789477110.237823,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2378478,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71","EoS":true} {"level":"info","ts":1789477110.2378576,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bd88e874676d452be0ff82ace3aad971","span_id":"eed2f4b4f3338226","x-request-id":"e1ec9db5-f522-4832-be70-fe8e48948e71"} {"level":"info","ts":1789477110.2782526,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477110.278354,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"info","ts":1789477110.278367,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"info","ts":1789477110.278441,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","EoS":true} {"level":"info","ts":1789477110.278471,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","header":"x-maas-group"} {"level":"info","ts":1789477110.2784817,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","header":"x-maas-subscription"} {"level":"info","ts":1789477110.278487,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","header":"x-maas-username"} {"level":"info","ts":1789477110.2785017,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789477110.2785263,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"info","ts":1789477110.2793171,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"info","ts":1789477110.279337,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"info","ts":1789477110.2793508,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","EoS":false} {"level":"info","ts":1789477110.2793643,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"info","ts":1789477110.279403,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c","EoS":true} {"level":"info","ts":1789477110.2794137,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3d3f4a4c98ed516064a96202f4a338a2","span_id":"2fa58ffb5f6783f7","x-request-id":"bca58cac-2cbf-4ab7-aa89-c79927112e8c"} {"level":"error","ts":1789477125.0094776,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477447.0785344,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477447.0786421,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.0786543,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.078666,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","EoS":true} {"level":"info","ts":1789477447.0786903,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","header":"x-maas-username"} {"level":"info","ts":1789477447.0787,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","header":"x-maas-group"} {"level":"info","ts":1789477447.0787048,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","header":"x-maas-subscription"} {"level":"info","ts":1789477447.0787199,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477447.0787456,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.091559,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.0915859,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.0916018,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","EoS":false} {"level":"info","ts":1789477447.0916135,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.0916371,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9","EoS":true} {"level":"info","ts":1789477447.0916467,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"12166c3766181bb7d8492059aec704db","span_id":"f476faa78a3c1d15","x-request-id":"107b0080-3483-4989-b65b-37856cb5a8b9"} {"level":"info","ts":1789477447.233553,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477447.2336576,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.2336707,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.2336822,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","EoS":true} {"level":"info","ts":1789477447.233706,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","header":"x-maas-username"} {"level":"info","ts":1789477447.2337158,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","header":"x-maas-group"} {"level":"info","ts":1789477447.2337208,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","header":"x-maas-subscription"} {"level":"info","ts":1789477447.2337365,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477447.2337625,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.236097,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.2361176,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.236136,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","EoS":false} {"level":"info","ts":1789477447.2361522,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.236187,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94","EoS":true} {"level":"info","ts":1789477447.236198,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"95d4db3e60ce3ba3acf8cb29d2a4994c","span_id":"c72c31e12e11a927","x-request-id":"2e2d946f-1e2c-4daf-9384-bc8dd4909a94"} {"level":"info","ts":1789477447.3767598,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477447.376921,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.3769388,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.3769596,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","EoS":true} {"level":"info","ts":1789477447.3769927,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","header":"x-maas-subscription"} {"level":"info","ts":1789477447.377008,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","header":"x-maas-username"} {"level":"info","ts":1789477447.3770142,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","header":"x-maas-group"} {"level":"info","ts":1789477447.377037,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477447.3770745,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.3780546,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.3780756,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.3780873,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","EoS":false} {"level":"info","ts":1789477447.378098,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.3781211,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895","EoS":true} {"level":"info","ts":1789477447.3781304,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c137f446a990763cf1cac244d7733e04","span_id":"143b8b68c9bcde70","x-request-id":"68a26ffe-9712-42ef-a6bb-58d375167895"} {"level":"info","ts":1789477447.5168452,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477447.5169559,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.5169694,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.516981,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","EoS":true} {"level":"info","ts":1789477447.517011,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","header":"x-maas-username"} {"level":"info","ts":1789477447.5170262,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","header":"x-maas-subscription"} {"level":"info","ts":1789477447.517034,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","header":"x-maas-group"} {"level":"info","ts":1789477447.5170488,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477447.5170743,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.517999,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.5180173,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.5180373,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","EoS":false} {"level":"info","ts":1789477447.5180535,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.5180879,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a","EoS":true} {"level":"info","ts":1789477447.5181,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c005a2c6f698d2ee27ec7503d46891c7","span_id":"c04c2816a8863c89","x-request-id":"a9d887c7-a171-9070-88cc-7fd5dcddff3a"} {"level":"info","ts":1789477447.6529117,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477447.6530244,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"info","ts":1789477447.6530385,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"info","ts":1789477447.6530576,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","EoS":true} {"level":"info","ts":1789477447.6530943,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","header":"x-maas-username"} {"level":"info","ts":1789477447.6531081,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","header":"x-maas-subscription"} {"level":"info","ts":1789477447.6531157,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","header":"x-maas-group"} {"level":"info","ts":1789477447.6531377,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477447.653172,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"info","ts":1789477447.6542034,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"info","ts":1789477447.6542168,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"info","ts":1789477447.654234,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","EoS":false} {"level":"info","ts":1789477447.654248,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"info","ts":1789477447.6542878,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3","EoS":true} {"level":"info","ts":1789477447.6543,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dfadb1cecfd482592800dd9c45d05e3e","span_id":"4819715c1d161f6e","x-request-id":"bc60cedd-b7ba-4f28-bab4-62f0de8b54f3"} {"level":"error","ts":1789477460.048794,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477462.705263,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477462.7054086,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7054322,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7054522,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","EoS":true} {"level":"info","ts":1789477462.7054832,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","header":"x-maas-group"} {"level":"info","ts":1789477462.7054958,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","header":"x-maas-subscription"} {"level":"info","ts":1789477462.7055035,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","header":"x-maas-username"} {"level":"info","ts":1789477462.7055225,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477462.7055585,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7068064,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7068317,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7068517,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","EoS":false} {"level":"info","ts":1789477462.706864,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7068932,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2","EoS":true} {"level":"info","ts":1789477462.706907,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d596ab81b0f97507b5edb2d3a7af8f85","span_id":"c9ef7baf809e033c","x-request-id":"216ed6a1-72e2-4e01-a353-73786b49e5e2"} {"level":"info","ts":1789477462.7434642,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477462.7435582,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.7435713,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.743583,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","EoS":true} {"level":"info","ts":1789477462.743606,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","header":"x-maas-username"} {"level":"info","ts":1789477462.7436168,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","header":"x-maas-subscription"} {"level":"info","ts":1789477462.7436216,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","header":"x-maas-group"} {"level":"info","ts":1789477462.7436361,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477462.743659,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.7447543,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.7447782,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.7447999,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","EoS":false} {"level":"info","ts":1789477462.7448127,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.7448363,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818","EoS":true} {"level":"info","ts":1789477462.7448456,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c0f4316dabd8e64a6cad2875d5ec00b5","span_id":"9a5fc03c22eca2a0","x-request-id":"59e39d56-6f09-4f7a-aa29-5636c5111818"} {"level":"info","ts":1789477462.8547285,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477462.8548298,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb"} {"level":"info","ts":1789477462.8558314,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb"} {"level":"info","ts":1789477462.8558555,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb"} {"level":"info","ts":1789477462.8558695,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb","EoS":false} {"level":"info","ts":1789477462.8558843,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb"} {"level":"info","ts":1789477462.8559062,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb","EoS":true} {"level":"info","ts":1789477462.8559153,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"432914a9f830f6a20d47606d5c4d0ed2","span_id":"f87cc54b00360f32","x-request-id":"07c554de-411d-4d1c-8106-7750f8ac29cb"} {"level":"info","ts":1789477463.6364713,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477463.6365697,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477463.6365821,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477463.6365938,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","EoS":true} {"level":"info","ts":1789477463.6366198,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","header":"x-maas-username"} {"level":"info","ts":1789477463.6366296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","header":"x-maas-subscription"} {"level":"info","ts":1789477463.636635,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","header":"x-maas-group"} {"level":"info","ts":1789477463.636651,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477463.6366765,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477463.6375494,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477463.637569,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477463.637584,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","EoS":false} {"level":"info","ts":1789477463.6375978,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477463.6376238,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698","EoS":true} {"level":"info","ts":1789477463.6376336,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5eab6be076179b87fb3d60d35d237828","span_id":"6fe4052801795622","x-request-id":"9f4dc66b-e4a9-409a-a609-ab4a26d48698"} {"level":"info","ts":1789477464.6023998,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477464.6025085,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477464.6025221,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477464.602534,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","EoS":true} {"level":"info","ts":1789477464.6025574,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","header":"x-maas-username"} {"level":"info","ts":1789477464.6025681,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","header":"x-maas-group"} {"level":"info","ts":1789477464.6025736,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","header":"x-maas-subscription"} {"level":"info","ts":1789477464.602589,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477464.6026127,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477464.6034243,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477464.6034448,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477464.6034682,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","EoS":false} {"level":"info","ts":1789477464.6034794,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477464.6035044,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051","EoS":true} {"level":"info","ts":1789477464.6035137,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"32afd14fc8091ce027d25aeb31e487f1","span_id":"d0b271fd1d2a73cf","x-request-id":"689752d6-516e-41e0-b200-b197967c8051"} {"level":"info","ts":1789477465.3426769,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477465.3427827,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477465.3427966,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477465.342808,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","EoS":true} {"level":"info","ts":1789477465.3428326,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","header":"x-maas-group"} {"level":"info","ts":1789477465.3428397,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","header":"x-maas-subscription"} {"level":"info","ts":1789477465.342845,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","header":"x-maas-username"} {"level":"info","ts":1789477465.3428597,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477465.3428795,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477465.3436608,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477465.3436801,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477465.3436964,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","EoS":false} {"level":"info","ts":1789477465.3437114,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477465.343745,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d","EoS":true} {"level":"info","ts":1789477465.3437555,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1223aea8408ab2e7353cd2ccf8249a82","span_id":"fe23b6a09024f56e","x-request-id":"21617eb3-968b-4206-a426-99957000f56d"} {"level":"info","ts":1789477466.4227772,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477466.4228728,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"info","ts":1789477466.4228864,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"info","ts":1789477466.4228978,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","EoS":true} {"level":"info","ts":1789477466.422921,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","header":"x-maas-subscription"} {"level":"info","ts":1789477466.4229288,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","header":"x-maas-username"} {"level":"info","ts":1789477466.422935,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","header":"x-maas-group"} {"level":"info","ts":1789477466.4229493,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477466.4229703,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"info","ts":1789477466.4237065,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"info","ts":1789477466.4237268,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"info","ts":1789477466.4237447,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","EoS":false} {"level":"info","ts":1789477466.4237623,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"info","ts":1789477466.4237876,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84","EoS":true} {"level":"info","ts":1789477466.423796,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"584082a440347adb3f74f2e2798646b4","span_id":"8125b3946262895c","x-request-id":"67d449af-8bce-41d2-888e-d5a3001f2d84"} {"level":"error","ts":1789477475.049421,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789477490.0509357,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477495.3296995,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477495.3297982,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"info","ts":1789477495.329812,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"info","ts":1789477495.329824,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","EoS":true} {"level":"info","ts":1789477495.3298478,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","header":"x-maas-group"} {"level":"info","ts":1789477495.329859,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","header":"x-maas-username"} {"level":"info","ts":1789477495.3298647,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","header":"x-maas-subscription"} {"level":"info","ts":1789477495.329879,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477495.3299072,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"info","ts":1789477495.3307233,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"info","ts":1789477495.3307505,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"info","ts":1789477495.3307705,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","EoS":false} {"level":"info","ts":1789477495.3307846,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"info","ts":1789477495.3308096,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c","EoS":true} {"level":"info","ts":1789477495.3308194,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f2e64de5272054e6e7712f537e54e0d2","span_id":"116d43514bf0db33","x-request-id":"de59535d-57be-482d-965b-3563523d022c"} {"level":"error","ts":1789477510.0532112,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477586.9650178,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477586.9651248,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"info","ts":1789477586.9651377,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"info","ts":1789477586.9651499,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","EoS":true} {"level":"info","ts":1789477586.9651754,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","header":"x-maas-subscription"} {"level":"info","ts":1789477586.9651864,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","header":"x-maas-username"} {"level":"info","ts":1789477586.9651911,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","header":"x-maas-group"} {"level":"info","ts":1789477586.9652054,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789477586.9652288,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"info","ts":1789477586.9781408,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"info","ts":1789477586.9781737,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"info","ts":1789477586.9781904,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","EoS":false} {"level":"info","ts":1789477586.9782035,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"info","ts":1789477586.9782412,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168","EoS":true} {"level":"info","ts":1789477586.9782605,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"25fc623ca290942244d282e5ad914bc7","span_id":"30241b4981fe28c2","x-request-id":"7aaa44e7-6a91-435a-8226-580c4670e168"} {"level":"error","ts":1789477600.0635104,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477672.3553057,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477672.355433,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"info","ts":1789477672.3554447,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"info","ts":1789477672.355458,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","EoS":true} {"level":"info","ts":1789477672.355485,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","header":"x-maas-username"} {"level":"info","ts":1789477672.355495,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","header":"x-maas-subscription"} {"level":"info","ts":1789477672.3554993,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","header":"x-maas-group"} {"level":"info","ts":1789477672.3555145,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789477672.355543,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"info","ts":1789477672.356485,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"info","ts":1789477672.3565052,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"info","ts":1789477672.356522,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","EoS":false} {"level":"info","ts":1789477672.35654,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"info","ts":1789477672.356564,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261","EoS":true} {"level":"info","ts":1789477672.3565729,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"70b243e46699411e6040628f7c52862d","span_id":"25b709fa3faf3932","x-request-id":"4a2badc6-e251-4468-9154-f89d1884c261"} {"level":"error","ts":1789477685.0721972,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477686.1004627,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477686.100566,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.1005785,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.1005907,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","EoS":true} {"level":"info","ts":1789477686.1006138,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","header":"x-maas-group"} {"level":"info","ts":1789477686.100622,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","header":"x-maas-subscription"} {"level":"info","ts":1789477686.1006265,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","header":"x-maas-username"} {"level":"info","ts":1789477686.1006422,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789477686.1006691,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.114201,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.1142297,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.114255,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","EoS":false} {"level":"info","ts":1789477686.1142688,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.1143076,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","EoS":false} {"level":"info","ts":1789477686.1143193,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.1143312,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900","EoS":true} {"level":"info","ts":1789477686.1143365,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a8512906be9f3df546d6ae462b0438dc","span_id":"889541358a68e48f","x-request-id":"0ab921d7-dbbc-40a6-948b-0de3ad112900"} {"level":"info","ts":1789477686.2526848,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477686.2527792,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.2527928,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.2528038,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","EoS":true} {"level":"info","ts":1789477686.2528253,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","header":"x-maas-subscription"} {"level":"info","ts":1789477686.2528346,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","header":"x-maas-username"} {"level":"info","ts":1789477686.2528389,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","header":"x-maas-group"} {"level":"info","ts":1789477686.252855,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789477686.2528806,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.255074,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.2550938,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.2551146,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","EoS":false} {"level":"info","ts":1789477686.2551322,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.2551703,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","EoS":false} {"level":"info","ts":1789477686.2551827,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.2552016,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716","EoS":true} {"level":"info","ts":1789477686.2552118,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c2f495fd7e30922c974f94d1a94a9e70","span_id":"792378cdaf14f317","x-request-id":"8210686c-2862-47af-acc6-9f39d24d7716"} {"level":"info","ts":1789477686.3962007,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477686.3963304,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.3963504,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.3963714,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","EoS":true} {"level":"info","ts":1789477686.3964317,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","header":"x-maas-username"} {"level":"info","ts":1789477686.3964443,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","header":"x-maas-subscription"} {"level":"info","ts":1789477686.396453,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","header":"x-maas-group"} {"level":"info","ts":1789477686.3964748,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789477686.3965077,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.3974679,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.39749,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.3975146,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","EoS":false} {"level":"info","ts":1789477686.397526,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.3975542,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","EoS":false} {"level":"info","ts":1789477686.3975666,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"info","ts":1789477686.3975918,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb","EoS":true} {"level":"info","ts":1789477686.3976016,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f1bcde2e561191391012e8779c7d091","span_id":"f44f80f70cbca790","x-request-id":"2cc804bb-9a8d-4672-b074-697cd978afdb"} {"level":"error","ts":1789477700.0734017,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789477700.2467709,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789477700.2468731,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"} {"level":"info","ts":1789477700.2468855,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"} {"level":"info","ts":1789477700.246898,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","EoS":true} {"level":"info","ts":1789477700.2469196,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","header":"x-maas-username"} {"level":"info","ts":1789477700.2469294,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","header":"x-maas-group"} {"level":"info","ts":1789477700.246935,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","header":"x-maas-subscription"} {"level":"info","ts":1789477700.2469537,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789477700.2469885,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"} {"level":"info","ts":1789477700.2487483,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"} {"level":"info","ts":1789477700.2487688,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"} {"level":"info","ts":1789477700.2487843,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","EoS":false} {"level":"info","ts":1789477700.248797,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"} {"level":"info","ts":1789477700.2488222,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1","EoS":true} {"level":"info","ts":1789477700.2488344,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9a7f3d3b4fb30035bbaa3786a66d5737","span_id":"de854138bbb533f4","x-request-id":"ba0124c6-b2c4-4380-82a0-a2cbccc74de1"}