{"level":"info","ts":1789503694.6176696,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c","header":"x-maas-subscription"} {"level":"info","ts":1789503694.617679,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c","header":"x-maas-username"} {"level":"info","ts":1789503694.6177008,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789503694.6177711,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c"} {"level":"info","ts":1789503694.6188354,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c"} {"level":"info","ts":1789503694.618855,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c"} {"level":"info","ts":1789503694.6188698,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c","EoS":false} {"level":"info","ts":1789503694.6188817,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c"} {"level":"info","ts":1789503694.6189075,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c","EoS":true} {"level":"info","ts":1789503694.618918,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4139fbb2de31774d54b6e0686b82c088","span_id":"4aa419c3385fc269","x-request-id":"5dc8b74e-a0ed-49a0-9666-bd8a8602061c"} {"level":"info","ts":1789503694.6825552,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503694.6826594,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503694.682672,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503694.6826837,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","EoS":true} {"level":"info","ts":1789503694.6827102,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","header":"x-maas-username"} {"level":"info","ts":1789503694.6827197,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","header":"x-maas-group"} {"level":"info","ts":1789503694.6827435,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","header":"x-maas-subscription"} {"level":"info","ts":1789503694.682762,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789503694.6827936,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503694.6845028,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503694.6845202,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503694.6845315,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","EoS":false} {"level":"info","ts":1789503694.6845417,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503694.6845636,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110","EoS":true} {"level":"info","ts":1789503694.6845734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4201df8d9e2985c8450d21efac90f47d","span_id":"612f4bdf4dc3f1fe","x-request-id":"ed57fdf0-784b-4c9f-bc5b-62e0f4887110"} {"level":"info","ts":1789503696.32252,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"5e7743f2-bd81-4cb0-b1fb-d42d6ad41b4a","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789503696.3225834,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"5e7743f2-bd81-4cb0-b1fb-d42d6ad41b4a","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789503696.322546,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"c106e6e4-954f-414d-891c-32bf34bdcf06"} {"level":"info","ts":1789503700.6232128,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503700.6233413,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c82a15ef1165824657f6a56f75fdf1d7","span_id":"a80579abd7741599","x-request-id":"30b847db-8328-420b-b357-2c8e735770c5"} {"level":"info","ts":1789503700.6233566,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c82a15ef1165824657f6a56f75fdf1d7","span_id":"a80579abd7741599","x-request-id":"30b847db-8328-420b-b357-2c8e735770c5"} {"level":"info","ts":1789503700.6233766,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c82a15ef1165824657f6a56f75fdf1d7","span_id":"a80579abd7741599","x-request-id":"30b847db-8328-420b-b357-2c8e735770c5","EoS":true} {"level":"info","ts":1789503700.6234355,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c82a15ef1165824657f6a56f75fdf1d7","span_id":"a80579abd7741599","x-request-id":"30b847db-8328-420b-b357-2c8e735770c5","path":"/llm/nonexistent-model-xyz/v1/chat/completions"} {"level":"info","ts":1789503700.623474,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c82a15ef1165824657f6a56f75fdf1d7","span_id":"a80579abd7741599","x-request-id":"30b847db-8328-420b-b357-2c8e735770c5"} {"level":"error","ts":1789503700.8939073,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789503700.9685988,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6b3cf8ab-bb5a-42d3-a403-10bb833ecf50","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.9725142,"caller":"legacymigration/reconciler.go:147","msg":"creating ExternalProvider","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6b3cf8ab-bb5a-42d3-a403-10bb833ecf50","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503700.9811945,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e5339588-c44d-4314-b12a-bc5e47075396","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.9813292,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e5339588-c44d-4314-b12a-bc5e47075396","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789503700.9814713,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a7e6d6ab-e680-4922-9852-ccc469a86e18"} {"level":"info","ts":1789503700.9912019,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"d0b1a16c-0b88-409c-aa50-099ee9b11fd1","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.9912455,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"d0b1a16c-0b88-409c-aa50-099ee9b11fd1","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"info","ts":1789503700.9916217,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"1bb8aa48-6a39-41b7-8ba1-efa4eed91587","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.9916482,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"1bb8aa48-6a39-41b7-8ba1-efa4eed91587","provider":"openai","endpoint":"httpbingo.org"} {"level":"error","ts":1789503700.9918017,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6b3cf8ab-bb5a-42d3-a403-10bb833ecf50","error":"failed to apply ExternalProvider: externalproviders.inference.opendatahub.io \"e2e-legacy-migration-501eef72\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503700.9918923,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2216a034-c7ce-4d36-951b-2456ec12f507"} {"level":"info","ts":1789503700.9922016,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e2e2c68c-1323-4dcf-a043-a0c7c15a9d18","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.992227,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e2e2c68c-1323-4dcf-a043-a0c7c15a9d18","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"info","ts":1789503700.9980898,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"369c9f6a-da63-4de1-b022-3fe8ae246c4a","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.9982708,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"f10f8ddd-7b64-4277-ad3e-d8c26d98b915","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503700.998303,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"f10f8ddd-7b64-4277-ad3e-d8c26d98b915","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"error","ts":1789503700.9984097,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a7e6d6ab-e680-4922-9852-ccc469a86e18","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:84\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503700.9984782,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a7e6d6ab-e680-4922-9852-ccc469a86e18","name":"e2e-legacy-migration-501eef72"} {"level":"error","ts":1789503701.007068,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2216a034-c7ce-4d36-951b-2456ec12f507","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.0071352,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2216a034-c7ce-4d36-951b-2456ec12f507","provider":"e2e-legacy-migration-501eef72","phase":"Pending"} {"level":"info","ts":1789503701.0122209,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"369c9f6a-da63-4de1-b022-3fe8ae246c4a","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.0124817,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"d87bbd32-588c-41b4-8fcf-027e12690322","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.012569,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"d87bbd32-588c-41b4-8fcf-027e12690322","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"error","ts":1789503701.0149875,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2216a034-c7ce-4d36-951b-2456ec12f507","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789503701.0150874,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2216a034-c7ce-4d36-951b-2456ec12f507","error":"ExternalModel \"e2e-legacy-migration-501eef72\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-501eef72\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.0152287,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a8bf8d8e-e436-49ee-a4ff-619973930c11"} {"level":"info","ts":1789503701.015305,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a8bf8d8e-e436-49ee-a4ff-619973930c11","provider":"e2e-legacy-migration-501eef72","phase":"Pending"} {"level":"error","ts":1789503701.0241895,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a8bf8d8e-e436-49ee-a4ff-619973930c11","error":"ExternalModel \"e2e-legacy-migration-501eef72\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-501eef72\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789503701.0256495,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a7e6d6ab-e680-4922-9852-ccc469a86e18","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789503701.0257106,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"a7e6d6ab-e680-4922-9852-ccc469a86e18","error":"failed to apply Service: services \"e2e-legacy-migration-501eef72\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.0258405,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e8ccc56b-bbd1-48ca-83b1-6728df08db94"} {"level":"info","ts":1789503701.0259166,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e8ccc56b-bbd1-48ca-83b1-6728df08db94","name":"e2e-legacy-migration-501eef72"} {"level":"error","ts":1789503701.027274,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"369c9f6a-da63-4de1-b022-3fe8ae246c4a","error":"failed to apply ExternalModel: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.0349681,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"58cf8bc4-f1c9-40b5-9e50-1944d80f528d"} {"level":"info","ts":1789503701.035013,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"58cf8bc4-f1c9-40b5-9e50-1944d80f528d","provider":"e2e-legacy-migration-501eef72","phase":"Pending"} {"level":"info","ts":1789503701.03752,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"430e4219-9509-4b83-9acc-5334b7d907e1","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.0411577,"caller":"externalprovider/reconciler.go:210","msg":"Creating resource","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e8ccc56b-bbd1-48ca-83b1-6728df08db94","kind":"DestinationRule","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.0413117,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"430e4219-9509-4b83-9acc-5334b7d907e1","name":"e2e-legacy-migration-501eef72"} {"level":"error","ts":1789503701.0461671,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"58cf8bc4-f1c9-40b5-9e50-1944d80f528d","error":"ExternalModel \"e2e-legacy-migration-501eef72\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-501eef72\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.0498145,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"430e4219-9509-4b83-9acc-5334b7d907e1","provider":"openai","model":"e2e-legacy-migration-501eef72","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789503701.065082,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"9da5c097-c2a5-44e2-b8de-bec1a87b5286","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.0651171,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"9da5c097-c2a5-44e2-b8de-bec1a87b5286","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789503701.0651267,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"0baecd02-4b20-41f6-8a6d-c9c13e171e3d","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.0651572,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"0baecd02-4b20-41f6-8a6d-c9c13e171e3d","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"info","ts":1789503701.0651834,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6ec2dce6-727f-4f5f-a1fb-e37b56fa6176"} {"level":"info","ts":1789503701.0652182,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6ec2dce6-727f-4f5f-a1fb-e37b56fa6176","name":"e2e-legacy-migration-501eef72"} {"level":"error","ts":1789503701.068484,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e8ccc56b-bbd1-48ca-83b1-6728df08db94","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789503701.0685294,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e8ccc56b-bbd1-48ca-83b1-6728df08db94","error":"failed to apply DestinationRule: destinationrules.networking.istio.io \"e2e-legacy-migration-501eef72\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.0685759,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"24e8bbcd-a7dd-48a4-b610-279dfdaf53f9"} {"level":"info","ts":1789503701.0686157,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"24e8bbcd-a7dd-48a4-b610-279dfdaf53f9","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.0949001,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"c7a50d8e-543b-47cf-a369-3569e4acd194","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.094953,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"c7a50d8e-543b-47cf-a369-3569e4acd194","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"error","ts":1789503701.099381,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6ec2dce6-727f-4f5f-a1fb-e37b56fa6176","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789503701.0994463,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"6ec2dce6-727f-4f5f-a1fb-e37b56fa6176","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-501eef72\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.099527,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"ff95bdf9-4261-42e5-9b29-c10388a4d4be"} {"level":"info","ts":1789503701.0996058,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"ff95bdf9-4261-42e5-9b29-c10388a4d4be","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.1069274,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"24e8bbcd-a7dd-48a4-b610-279dfdaf53f9","service":"e2e-legacy-migration-501eef72","serviceEntry":"e2e-legacy-migration-501eef72","destinationRule":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.1105323,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"cfcf5318-796d-4c2b-9dcd-f118c899ed53","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.110577,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"cfcf5318-796d-4c2b-9dcd-f118c899ed53","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"info","ts":1789503701.1282356,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"ff95bdf9-4261-42e5-9b29-c10388a4d4be","httpRoute":"e2e-legacy-migration-501eef72","providers":["e2e-legacy-migration-501eef72"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789503701.1515403,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"ff95bdf9-4261-42e5-9b29-c10388a4d4be","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.1516411,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e3b9e20f-36b7-4bbf-8333-1864e7a9982e"} {"level":"info","ts":1789503701.151705,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e3b9e20f-36b7-4bbf-8333-1864e7a9982e","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.1869485,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e3b9e20f-36b7-4bbf-8333-1864e7a9982e","httpRoute":"e2e-legacy-migration-501eef72","providers":["e2e-legacy-migration-501eef72"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789503701.1912744,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"3d92e660-5a92-468b-9766-bfbf88460689","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.1913297,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"3d92e660-5a92-468b-9766-bfbf88460689","modelName":"e2e-legacy-migration-501eef72","resolvedRefs":1} {"level":"error","ts":1789503701.1950383,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"e3b9e20f-36b7-4bbf-8333-1864e7a9982e","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503701.1951482,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"afbd2e9a-fa4f-4d6d-a25d-230459fbd0d5"} {"level":"info","ts":1789503701.1952188,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"afbd2e9a-fa4f-4d6d-a25d-230459fbd0d5","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.2156296,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"afbd2e9a-fa4f-4d6d-a25d-230459fbd0d5","httpRoute":"e2e-legacy-migration-501eef72","providers":["e2e-legacy-migration-501eef72"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789503701.3111813,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"8b2d2678-84ff-4cb7-a31e-a9c611db373b","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503701.3163478,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"8b2d2678-84ff-4cb7-a31e-a9c611db373b","name":"e2e-legacy-migration-501eef72"} {"level":"info","ts":1789503701.3249853,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"8b2d2678-84ff-4cb7-a31e-a9c611db373b","provider":"openai","model":"e2e-legacy-migration-501eef72","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789503703.9251504,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"c9a86491-7a7b-431e-9a9b-80ca733391df","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503703.932893,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2569fb93-cd57-41cc-bf7f-28a45f167b9f","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503703.9329376,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"2569fb93-cd57-41cc-bf7f-28a45f167b9f","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503703.9329786,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"4644f870-dd31-4e1d-96f5-70f8f3e7047a","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503703.933011,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"30f00038-f0d3-4f8d-9447-619cf857df00"} {"level":"error","ts":1789503703.9330077,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"4644f870-dd31-4e1d-96f5-70f8f3e7047a","provider":"e2e-legacy-migration-501eef72","error":"ExternalProvider \"e2e-legacy-migration-501eef72\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503703.9330277,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"4644f870-dd31-4e1d-96f5-70f8f3e7047a"} {"level":"error","ts":1789503703.9330308,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"30f00038-f0d3-4f8d-9447-619cf857df00","provider":"e2e-legacy-migration-501eef72","error":"ExternalProvider.inference.opendatahub.io \"e2e-legacy-migration-501eef72\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503703.9330812,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"04844332-1f58-4f57-8764-9ac2a0db721c"} {"level":"info","ts":1789503703.9342065,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"1c671ec9-d8ce-4fd8-bf75-88d6b9e9a815","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"info","ts":1789503703.934232,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"1c671ec9-d8ce-4fd8-bf75-88d6b9e9a815","name":"e2e-legacy-migration-501eef72","namespace":"llm"} {"level":"error","ts":1789503703.9416375,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"30f00038-f0d3-4f8d-9447-619cf857df00","error":"externalmodels.inference.opendatahub.io \"e2e-legacy-migration-501eef72\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789503703.9416957,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"30f00038-f0d3-4f8d-9447-619cf857df00","error":"ExternalModel \"e2e-legacy-migration-501eef72\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-501eef72\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789503703.941938,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-501eef72","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-501eef72","reconcileID":"fc281d1c-1a86-4ca9-8809-1708c52ca198"} {"level":"info","ts":1789503704.3779216,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503704.378056,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.378073,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.3780918,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","EoS":true} {"level":"info","ts":1789503704.3781257,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","header":"x-maas-username"} {"level":"info","ts":1789503704.378137,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","header":"x-maas-group"} {"level":"info","ts":1789503704.3781452,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","header":"x-maas-subscription"} {"level":"info","ts":1789503704.3781662,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789503704.3781824,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","modelName":"e2e-external-model"} {"level":"info","ts":1789503704.378199,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789503704.3782272,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789503704.3782513,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","authType":"apikey"} {"level":"info","ts":1789503704.3782792,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.4252496,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.4253447,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.4253912,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","EoS":false} {"level":"info","ts":1789503704.4254262,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.425492,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317","EoS":true} {"level":"info","ts":1789503704.425517,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"883f4e4872dd3bca2955acaeb2854e99","span_id":"5932f9920b1a04ca","x-request-id":"e93eddc8-f448-462d-a48f-7b83e012e317"} {"level":"info","ts":1789503704.4776113,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503704.4779768,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.4780583,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.4781048,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","EoS":true} {"level":"info","ts":1789503704.47818,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","header":"x-maas-subscription"} {"level":"info","ts":1789503704.478216,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","header":"x-maas-group"} {"level":"info","ts":1789503704.4782634,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","header":"x-maas-username"} {"level":"info","ts":1789503704.4783049,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789503704.4783747,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.5193477,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.5193758,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.519391,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","EoS":false} {"level":"info","ts":1789503704.5194068,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.5194354,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f","EoS":true} {"level":"info","ts":1789503704.519443,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"144b931eaadbb175a227669eb72e5f30","span_id":"f835a1ae9a9acb78","x-request-id":"a663aa44-960c-4285-abd7-d2dd7872391f"} {"level":"info","ts":1789503704.5789392,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503704.5791695,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503704.5792255,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503704.5792687,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a","EoS":true} {"level":"info","ts":1789503704.5793421,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a","header":"x-maas-group"} {"level":"info","ts":1789503704.5793796,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a","header":"x-maas-subscription"} {"level":"info","ts":1789503704.5794256,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a","header":"x-maas-username"} {"level":"info","ts":1789503704.579498,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503704.623272,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503704.6233203,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503704.6233382,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a","EoS":false} {"level":"info","ts":1789503704.623352,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503704.6233761,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a","EoS":true} {"level":"info","ts":1789503704.623386,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f203b4537bd23e82fcdf8e0ce83ec29b","span_id":"e5163011c35816c3","x-request-id":"455bb578-4c73-4ef4-9655-ad4e5c33749a"} {"level":"info","ts":1789503705.5232546,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"335a6f21-4209-434d-a58f-7d35fe2a3d98","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789503705.5233104,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"335a6f21-4209-434d-a58f-7d35fe2a3d98","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789503705.523274,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"3f74a35f-5709-41e4-99c1-14ae0511964f"} {"level":"info","ts":1789503705.6626709,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"8902dd3b-7b38-4ca4-8bf3-dc45bb90c806"} {"level":"info","ts":1789503705.6626706,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"7e63dff0-3ffc-4cbd-a805-138715e2f926","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789503705.6627624,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"7e63dff0-3ffc-4cbd-a805-138715e2f926","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789503705.7758586,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"15ff6652-0164-4814-9c6a-06d94cd57cd0","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789503705.7761164,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"15ff6652-0164-4814-9c6a-06d94cd57cd0","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789503710.8959591,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789503712.58909,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503712.5892303,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"82f2d077bdaab703e041db1b1b1eddcb","span_id":"c7e4a472dff97fa4","x-request-id":"c0832387-bd26-4dca-a2a0-fb82eb235fb2"} {"level":"info","ts":1789503712.5892487,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"82f2d077bdaab703e041db1b1b1eddcb","span_id":"c7e4a472dff97fa4","x-request-id":"c0832387-bd26-4dca-a2a0-fb82eb235fb2"} {"level":"info","ts":1789503712.589267,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"82f2d077bdaab703e041db1b1b1eddcb","span_id":"c7e4a472dff97fa4","x-request-id":"c0832387-bd26-4dca-a2a0-fb82eb235fb2","EoS":true} {"level":"info","ts":1789503712.5893655,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"82f2d077bdaab703e041db1b1b1eddcb","span_id":"c7e4a472dff97fa4","x-request-id":"c0832387-bd26-4dca-a2a0-fb82eb235fb2"} {"level":"info","ts":1789503712.7432742,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503712.7433808,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3a3edbaf96a93cd3cacad78a634d5d0f","span_id":"d4e7f7775066e057","x-request-id":"3840c041-96b9-4a55-85a1-0688bac3b2b1"} {"level":"info","ts":1789503712.743395,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3a3edbaf96a93cd3cacad78a634d5d0f","span_id":"d4e7f7775066e057","x-request-id":"3840c041-96b9-4a55-85a1-0688bac3b2b1"} {"level":"info","ts":1789503712.7434068,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3a3edbaf96a93cd3cacad78a634d5d0f","span_id":"d4e7f7775066e057","x-request-id":"3840c041-96b9-4a55-85a1-0688bac3b2b1","EoS":true} {"level":"info","ts":1789503712.743453,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3a3edbaf96a93cd3cacad78a634d5d0f","span_id":"d4e7f7775066e057","x-request-id":"3840c041-96b9-4a55-85a1-0688bac3b2b1"} {"level":"info","ts":1789503712.888948,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789503712.8891838,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e8b077242affe183a1eb02150677aa26","span_id":"db617bae62246484","x-request-id":"d901aaa7-ad0d-4895-ac30-f9f3a3351da7"} {"level":"info","ts":1789503712.889236,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e8b077242affe183a1eb02150677aa26","span_id":"db617bae62246484","x-request-id":"d901aaa7-ad0d-4895-ac30-f9f3a3351da7"} {"level":"info","ts":1789503712.889288,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e8b077242affe183a1eb02150677aa26","span_id":"db617bae62246484","x-request-id":"d901aaa7-ad0d-4895-ac30-f9f3a3351da7","EoS":true} {"level":"info","ts":1789503712.8893688,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e8b077242affe183a1eb02150677aa26","span_id":"db617bae62246484","x-request-id":"d901aaa7-ad0d-4895-ac30-f9f3a3351da7"} {"level":"error","ts":1789503725.8994844,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789504817.8750987,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789504817.875221,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.875236,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.87525,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","EoS":true} {"level":"info","ts":1789504817.8752778,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","header":"x-maas-username"} {"level":"info","ts":1789504817.8752923,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","header":"x-maas-subscription"} {"level":"info","ts":1789504817.8752985,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","header":"x-maas-group"} {"level":"info","ts":1789504817.8753145,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789504817.8753436,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.8765485,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.8765757,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.8765917,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","EoS":false} {"level":"info","ts":1789504817.876609,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.8766334,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377","EoS":true} {"level":"info","ts":1789504817.8766434,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b4b9a462c342faf553e1fc5ef0ab4e95","span_id":"ef19355efe227a31","x-request-id":"4e4fb6a2-b0eb-417d-90f3-82eb04b4d377"} {"level":"info","ts":1789504817.9108124,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789504817.9109359,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"info","ts":1789504817.9109528,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"info","ts":1789504817.9109654,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","EoS":true} {"level":"info","ts":1789504817.9109979,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","header":"x-maas-subscription"} {"level":"info","ts":1789504817.9110134,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","header":"x-maas-username"} {"level":"info","ts":1789504817.9110198,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","header":"x-maas-group"} {"level":"info","ts":1789504817.9110348,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789504817.9110622,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"info","ts":1789504817.9120345,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"info","ts":1789504817.9120603,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"info","ts":1789504817.9120753,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","EoS":false} {"level":"info","ts":1789504817.9120886,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"info","ts":1789504817.9121132,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a","EoS":true} {"level":"info","ts":1789504817.9121222,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16152de513228870671fbe15a1bf6b4f","span_id":"0a56c1c8a59a0076","x-request-id":"cff7cecb-b79a-479e-8c34-575c0259368a"} {"level":"error","ts":1789504831.029225,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505135.0115232,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505135.0116441,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.0116603,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.0116735,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","EoS":true} {"level":"info","ts":1789505135.011698,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","header":"x-maas-username"} {"level":"info","ts":1789505135.0117059,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","header":"x-maas-group"} {"level":"info","ts":1789505135.0117116,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","header":"x-maas-subscription"} {"level":"info","ts":1789505135.0117455,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505135.0118191,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.0246713,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.0247009,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.0247173,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","EoS":false} {"level":"info","ts":1789505135.0247493,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.0247774,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b","EoS":true} {"level":"info","ts":1789505135.024788,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3e1bd54f031b83130bf484ebd6e9442c","span_id":"39fbdd8bf2c35d4d","x-request-id":"4f6602d1-87be-4146-bc67-f1e6d7bb1f5b"} {"level":"info","ts":1789505135.1788297,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505135.1789353,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.178947,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.1789596,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","EoS":true} {"level":"info","ts":1789505135.1789832,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","header":"x-maas-username"} {"level":"info","ts":1789505135.1789942,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","header":"x-maas-subscription"} {"level":"info","ts":1789505135.1789982,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","header":"x-maas-group"} {"level":"info","ts":1789505135.1790128,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505135.1790395,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.1799018,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.1799235,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.1799378,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","EoS":false} {"level":"info","ts":1789505135.1799479,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.1799693,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a","EoS":true} {"level":"info","ts":1789505135.1799786,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3c72406542b0967f2ff87e8ca943cc71","span_id":"2eefb927f13e2189","x-request-id":"9d050389-45fb-4ce6-a077-76e2decd429a"} {"level":"info","ts":1789505135.3274972,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505135.327664,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.3276844,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.3276997,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","EoS":true} {"level":"info","ts":1789505135.3277533,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","header":"x-maas-group"} {"level":"info","ts":1789505135.3277647,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","header":"x-maas-subscription"} {"level":"info","ts":1789505135.327771,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","header":"x-maas-username"} {"level":"info","ts":1789505135.3277864,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505135.327814,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.32962,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.3296363,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.3296456,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","EoS":false} {"level":"info","ts":1789505135.3296564,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.329674,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da","EoS":true} {"level":"info","ts":1789505135.3296828,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"83d819efeb2fa22f170545ea1239a38a","span_id":"cdfad62f8f98a6dc","x-request-id":"53d307e6-7b2a-4b45-b239-6166bbd946da"} {"level":"info","ts":1789505135.4733107,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505135.4734132,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.4734273,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.4734395,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","EoS":true} {"level":"info","ts":1789505135.4734683,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","header":"x-maas-username"} {"level":"info","ts":1789505135.4734826,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","header":"x-maas-group"} {"level":"info","ts":1789505135.4734895,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","header":"x-maas-subscription"} {"level":"info","ts":1789505135.4735072,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505135.473535,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.474329,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.4743495,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.4743645,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","EoS":false} {"level":"info","ts":1789505135.4743752,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.4744031,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312","EoS":true} {"level":"info","ts":1789505135.4744174,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f07d73eb6c380b7250d63706f5a49793","span_id":"ea3c8a8ccf07dc48","x-request-id":"c306e9bb-61cb-4a63-af2e-4535856c2312"} {"level":"info","ts":1789505135.6141336,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505135.6142333,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505135.6142454,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505135.614257,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","EoS":true} {"level":"info","ts":1789505135.6142848,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","header":"x-maas-username"} {"level":"info","ts":1789505135.6142945,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","header":"x-maas-subscription"} {"level":"info","ts":1789505135.6143005,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","header":"x-maas-group"} {"level":"info","ts":1789505135.6143155,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505135.6143422,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505135.6152916,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505135.6153107,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505135.6153355,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","EoS":false} {"level":"info","ts":1789505135.615351,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505135.615379,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e","EoS":true} {"level":"info","ts":1789505135.6153927,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"241c869316f3731b1774550ac04b6658","span_id":"0eebe71c53aaa79f","x-request-id":"0c8835da-ad36-4ce6-8e93-1daeb2e7d26e"} {"level":"info","ts":1789505144.515392,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505144.5154893,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.515502,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.5155134,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","EoS":true} {"level":"info","ts":1789505144.515538,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","header":"x-maas-username"} {"level":"info","ts":1789505144.5155478,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","header":"x-maas-subscription"} {"level":"info","ts":1789505144.5155532,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","header":"x-maas-group"} {"level":"info","ts":1789505144.5155756,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505144.5155952,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.5164866,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.516513,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.5165274,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","EoS":false} {"level":"info","ts":1789505144.5165405,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.516569,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd","EoS":true} {"level":"info","ts":1789505144.5165777,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01e7e062847d651970d9e8d716d9ba9a","span_id":"77415f084162df37","x-request-id":"d56cb90b-ce4d-4164-b06e-42ba032053fd"} {"level":"info","ts":1789505144.635503,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505144.6356091,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb"} {"level":"info","ts":1789505144.6364505,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb"} {"level":"info","ts":1789505144.6364706,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb"} {"level":"info","ts":1789505144.6364892,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb","EoS":false} {"level":"info","ts":1789505144.6365013,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb"} {"level":"info","ts":1789505144.6365273,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb","EoS":true} {"level":"info","ts":1789505144.6365368,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8a1065e1392feb631f93e13cf3bb7206","span_id":"a1b85e83c12aed18","x-request-id":"8637588c-8252-47aa-b5b8-7d66ac28ccdb"} {"level":"info","ts":1789505145.403241,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505145.4033453,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"info","ts":1789505145.4033575,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"info","ts":1789505145.403369,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","EoS":true} {"level":"info","ts":1789505145.403396,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","header":"x-maas-group"} {"level":"info","ts":1789505145.4034057,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","header":"x-maas-username"} {"level":"info","ts":1789505145.4034107,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","header":"x-maas-subscription"} {"level":"info","ts":1789505145.4034243,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505145.4034543,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"info","ts":1789505145.4045362,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"info","ts":1789505145.4045596,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"info","ts":1789505145.4045758,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","EoS":false} {"level":"info","ts":1789505145.404587,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"info","ts":1789505145.404612,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10","EoS":true} {"level":"info","ts":1789505145.4046187,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fd90b85c6b81271339255249449e2ee5","span_id":"64bb2856d6fc33eb","x-request-id":"fbb00e93-3ee3-4f28-b54a-d749779fdb10"} {"level":"error","ts":1789505146.0652397,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505148.4781694,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505148.478281,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505148.478297,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505148.4783094,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","EoS":true} {"level":"info","ts":1789505148.4783347,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","header":"x-maas-username"} {"level":"info","ts":1789505148.478383,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","header":"x-maas-subscription"} {"level":"info","ts":1789505148.478392,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","header":"x-maas-group"} {"level":"info","ts":1789505148.4784079,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505148.4784372,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505148.4794092,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505148.479436,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505148.4794774,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","EoS":false} {"level":"info","ts":1789505148.4794943,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505148.4795315,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d","EoS":true} {"level":"info","ts":1789505148.4795454,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e084794abea93ca5cef3c9806469df7","span_id":"5da44c785c856891","x-request-id":"bbcaa208-02a6-434d-8c36-ec8b88ba823d"} {"level":"info","ts":1789505149.2204144,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505149.2205198,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505149.220533,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505149.220545,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","EoS":true} {"level":"info","ts":1789505149.2205708,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","header":"x-maas-subscription"} {"level":"info","ts":1789505149.2205813,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","header":"x-maas-username"} {"level":"info","ts":1789505149.220586,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","header":"x-maas-group"} {"level":"info","ts":1789505149.2206013,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505149.22063,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505149.2216384,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505149.2216582,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505149.2216747,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","EoS":false} {"level":"info","ts":1789505149.2216914,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505149.2217164,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815","EoS":true} {"level":"info","ts":1789505149.2217422,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a0f9ade9e0dac5b7869d3b246a2cc059","span_id":"33829af381ba4cb8","x-request-id":"ce10b7a5-d0ba-4822-81f5-2168b6442815"} {"level":"info","ts":1789505150.3604488,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505150.360562,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"info","ts":1789505150.3605762,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"info","ts":1789505150.360589,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","EoS":true} {"level":"info","ts":1789505150.360617,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","header":"x-maas-username"} {"level":"info","ts":1789505150.3606277,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","header":"x-maas-group"} {"level":"info","ts":1789505150.3606331,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","header":"x-maas-subscription"} {"level":"info","ts":1789505150.360649,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505150.3606763,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"info","ts":1789505150.3616054,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"info","ts":1789505150.3616347,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"info","ts":1789505150.361654,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","EoS":false} {"level":"info","ts":1789505150.361669,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"info","ts":1789505150.361695,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d","EoS":true} {"level":"info","ts":1789505150.361705,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a488dc48c8dafe416266f7934dbc79ad","span_id":"2df00c9dde6af6ec","x-request-id":"47c1f4a1-5f30-4566-8609-d2f4d84f7f1d"} {"level":"error","ts":1789505156.065808,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505168.3397179,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505168.3398514,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"info","ts":1789505168.3398654,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"info","ts":1789505168.339877,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","EoS":true} {"level":"info","ts":1789505168.3399022,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","header":"x-maas-group"} {"level":"info","ts":1789505168.3399124,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","header":"x-maas-username"} {"level":"info","ts":1789505168.3399181,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","header":"x-maas-subscription"} {"level":"info","ts":1789505168.3399334,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505168.3399591,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"info","ts":1789505168.3409393,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"info","ts":1789505168.3409607,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"info","ts":1789505168.3409753,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","EoS":false} {"level":"info","ts":1789505168.3409922,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"info","ts":1789505168.3410223,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4","EoS":true} {"level":"info","ts":1789505168.3410344,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"455e66cb3cb92a8ebd751370e1d084be","span_id":"cf0668d5f5034f6f","x-request-id":"b528dc99-fffe-46cf-8588-95fd250516f4"} {"level":"error","ts":1789505171.0675836,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789505186.0692995,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505262.0961108,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505262.0962152,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"info","ts":1789505262.0962286,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"info","ts":1789505262.096241,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","EoS":true} {"level":"info","ts":1789505262.0962636,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","header":"x-maas-group"} {"level":"info","ts":1789505262.0962741,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","header":"x-maas-subscription"} {"level":"info","ts":1789505262.0962796,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","header":"x-maas-username"} {"level":"info","ts":1789505262.096295,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789505262.0963218,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"info","ts":1789505262.109608,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"info","ts":1789505262.1096323,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"info","ts":1789505262.1096482,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","EoS":false} {"level":"info","ts":1789505262.1096606,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"info","ts":1789505262.1096883,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1","EoS":true} {"level":"info","ts":1789505262.1096973,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5455d5dcd40ab6e14dd8a29c16e18700","span_id":"986bb708736b92ef","x-request-id":"980d3df6-f2b2-429c-9be0-370088181fd1"} {"level":"error","ts":1789505276.0788836,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505354.904872,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505354.9049747,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"info","ts":1789505354.9049857,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"info","ts":1789505354.905001,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","EoS":true} {"level":"info","ts":1789505354.905031,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","header":"x-maas-username"} {"level":"info","ts":1789505354.9050393,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","header":"x-maas-group"} {"level":"info","ts":1789505354.9050443,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","header":"x-maas-subscription"} {"level":"info","ts":1789505354.9050596,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789505354.9050899,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"info","ts":1789505354.9069328,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"info","ts":1789505354.906964,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"info","ts":1789505354.9069843,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","EoS":false} {"level":"info","ts":1789505354.9070013,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"info","ts":1789505354.9070363,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3","EoS":true} {"level":"info","ts":1789505354.907048,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c7006f1c6ce9cd04bcaed811d23261a4","span_id":"6e22f280796ceeb6","x-request-id":"c8a92df2-3aca-4549-8a60-b7e03416f1e3"} {"level":"error","ts":1789505366.0877905,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505368.5972972,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505368.59743,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.5974505,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.5974681,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","EoS":true} {"level":"info","ts":1789505368.5974996,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","header":"x-maas-group"} {"level":"info","ts":1789505368.5975132,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","header":"x-maas-subscription"} {"level":"info","ts":1789505368.5975208,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","header":"x-maas-username"} {"level":"info","ts":1789505368.597541,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789505368.5975795,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.610045,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.610071,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.6100898,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","EoS":false} {"level":"info","ts":1789505368.6101038,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.6101341,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","EoS":false} {"level":"info","ts":1789505368.6101449,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.610156,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d","EoS":true} {"level":"info","ts":1789505368.610163,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cd5b22b25166e22fd579ba89960b66c3","span_id":"b56170fe5bae84e4","x-request-id":"6cd85fb6-6def-4ed0-a19a-b2be65e71a7d"} {"level":"info","ts":1789505368.7547076,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505368.75483,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.7548444,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.7548566,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","EoS":true} {"level":"info","ts":1789505368.7548804,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","header":"x-maas-subscription"} {"level":"info","ts":1789505368.7548919,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","header":"x-maas-username"} {"level":"info","ts":1789505368.7548969,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","header":"x-maas-group"} {"level":"info","ts":1789505368.7549121,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789505368.7549398,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.7568786,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.7568994,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.7569196,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","EoS":false} {"level":"info","ts":1789505368.756936,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.756974,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","EoS":false} {"level":"info","ts":1789505368.7569866,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.7570076,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572","EoS":true} {"level":"info","ts":1789505368.7570212,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9116f132ed499d16fd4d94f3cd11ccc5","span_id":"d2a87dfb73df1f21","x-request-id":"9b7562f6-f9c3-4ab8-9978-1903c15f7572"} {"level":"info","ts":1789505368.895992,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505368.8960836,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"info","ts":1789505368.8960958,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"info","ts":1789505368.8961077,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","EoS":true} {"level":"info","ts":1789505368.8961308,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","header":"x-maas-subscription"} {"level":"info","ts":1789505368.8961408,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","header":"x-maas-group"} {"level":"info","ts":1789505368.8961456,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","header":"x-maas-username"} {"level":"info","ts":1789505368.8961606,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789505368.8961847,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"info","ts":1789505368.8971019,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"info","ts":1789505368.8971226,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"info","ts":1789505368.8971407,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","EoS":false} {"level":"info","ts":1789505368.8971553,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"info","ts":1789505368.8971853,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555","EoS":true} {"level":"info","ts":1789505368.8971958,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fc803effb582002265ded46a897374b1","span_id":"fb33dd76edb6d10d","x-request-id":"98495804-c0b3-423a-a709-cb26d92b7555"} {"level":"error","ts":1789505381.0889213,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789505381.6733618,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789505381.6734743,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"} {"level":"info","ts":1789505381.6734862,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"} {"level":"info","ts":1789505381.673499,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","EoS":true} {"level":"info","ts":1789505381.67352,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","header":"x-maas-username"} {"level":"info","ts":1789505381.6735315,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","header":"x-maas-subscription"} {"level":"info","ts":1789505381.6735373,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","header":"x-maas-group"} {"level":"info","ts":1789505381.673552,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789505381.6735814,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"} {"level":"info","ts":1789505381.6753664,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"} {"level":"info","ts":1789505381.6753886,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"} {"level":"info","ts":1789505381.6754043,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","EoS":false} {"level":"info","ts":1789505381.6754148,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"} {"level":"info","ts":1789505381.6754386,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5","EoS":true} {"level":"info","ts":1789505381.6754458,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c1c433c11bca69d2ca524126003f0a6f","span_id":"62da93e2d16b6fcb","x-request-id":"48666e7e-921f-4284-8620-cc1ad80247a5"}