{"level":"info","ts":1789473185.0215547,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed","header":"x-maas-group"} {"level":"info","ts":1789473185.0215645,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed","header":"x-maas-username"} {"level":"info","ts":1789473185.0215702,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed","header":"x-maas-subscription"} {"level":"info","ts":1789473185.0215836,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789473185.0216072,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed"} {"level":"info","ts":1789473185.0225313,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed"} {"level":"info","ts":1789473185.022549,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed"} {"level":"info","ts":1789473185.0225618,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed","EoS":false} {"level":"info","ts":1789473185.0225728,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed"} {"level":"info","ts":1789473185.0225961,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed","EoS":true} {"level":"info","ts":1789473185.0226068,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fc7dfbabcf52296bef34c185cee86361","span_id":"1c60656556303e6c","x-request-id":"0433d165-ba09-483e-88b0-f2eea825b1ed"} {"level":"info","ts":1789473185.0640411,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473185.0641603,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473185.0641756,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473185.0641916,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","EoS":true} {"level":"info","ts":1789473185.0642197,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","header":"x-maas-username"} {"level":"info","ts":1789473185.0642304,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","header":"x-maas-group"} {"level":"info","ts":1789473185.0642383,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","header":"x-maas-subscription"} {"level":"info","ts":1789473185.0642617,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789473185.064287,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473185.0720062,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473185.0720308,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473185.072052,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","EoS":false} {"level":"info","ts":1789473185.072069,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473185.0721054,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc","EoS":true} {"level":"info","ts":1789473185.0721157,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a2c612f246c6bd08b12132d880a0cd53","span_id":"d102b6b8906fb054","x-request-id":"0f5dbe4c-91ed-488f-aae4-4e84224dfbcc"} {"level":"info","ts":1789473187.7563498,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"397cf4f1-a8a3-451a-8421-8c939d3044c6","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789473187.7563987,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"397cf4f1-a8a3-451a-8421-8c939d3044c6","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789473187.7563903,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"4d1cdcb3-83fe-48e5-928c-5966936c14f6"} {"level":"info","ts":1789473192.2723563,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473192.2725189,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7efab584de41e7cbe68a4fbee9ac9ed2","span_id":"bc8f94a1742eda1c","x-request-id":"fef513a4-bce1-4e77-b5d5-1638d3f8488f"} {"level":"info","ts":1789473192.2725382,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7efab584de41e7cbe68a4fbee9ac9ed2","span_id":"bc8f94a1742eda1c","x-request-id":"fef513a4-bce1-4e77-b5d5-1638d3f8488f"} {"level":"info","ts":1789473192.2725625,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7efab584de41e7cbe68a4fbee9ac9ed2","span_id":"bc8f94a1742eda1c","x-request-id":"fef513a4-bce1-4e77-b5d5-1638d3f8488f","EoS":true} {"level":"info","ts":1789473192.2726152,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7efab584de41e7cbe68a4fbee9ac9ed2","span_id":"bc8f94a1742eda1c","x-request-id":"fef513a4-bce1-4e77-b5d5-1638d3f8488f","path":"/llm/nonexistent-model-xyz/v1/chat/completions"} {"level":"info","ts":1789473192.2726524,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7efab584de41e7cbe68a4fbee9ac9ed2","span_id":"bc8f94a1742eda1c","x-request-id":"fef513a4-bce1-4e77-b5d5-1638d3f8488f"} {"level":"info","ts":1789473192.6514788,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"073093d0-3cb3-44ff-8220-b8ec545aae08","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.6582613,"caller":"legacymigration/reconciler.go:147","msg":"creating ExternalProvider","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"073093d0-3cb3-44ff-8220-b8ec545aae08","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.6623018,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"161002ca-1820-4d88-8504-9e58195c638c","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.6625674,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"161002ca-1820-4d88-8504-9e58195c638c","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789473192.662455,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"82acbc21-f97f-49d5-a9f3-c3bab28b3787"} {"level":"info","ts":1789473192.6735308,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"92c49add-16b7-4eff-af48-9a0d2bb68e70","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.6740777,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"92c49add-16b7-4eff-af48-9a0d2bb68e70","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"info","ts":1789473192.6739473,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"871a0b33-7131-4e47-9f27-eafafd5d11ce"} {"level":"info","ts":1789473192.6740036,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"644dd89b-c34f-4097-b694-90ba104e192c","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.6745424,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"644dd89b-c34f-4097-b694-90ba104e192c","provider":"openai","endpoint":"httpbingo.org"} {"level":"error","ts":1789473192.6737936,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"82acbc21-f97f-49d5-a9f3-c3bab28b3787","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:84\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.6747048,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"82acbc21-f97f-49d5-a9f3-c3bab28b3787","name":"e2e-legacy-migration-fcd350da"} {"level":"error","ts":1789473192.6751885,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"073093d0-3cb3-44ff-8220-b8ec545aae08","error":"failed to apply ExternalProvider: externalproviders.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.6782415,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"77d7ced0-c8c0-4843-81e8-fb8c39f08527","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.678311,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"77d7ced0-c8c0-4843-81e8-fb8c39f08527","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"info","ts":1789473192.6807644,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"2be1b699-1ceb-4df5-a688-721b4e5c1884","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"error","ts":1789473192.684199,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"871a0b33-7131-4e47-9f27-eafafd5d11ce","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.6842344,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"871a0b33-7131-4e47-9f27-eafafd5d11ce","provider":"e2e-legacy-migration-fcd350da","phase":"Pending"} {"level":"info","ts":1789473192.6877036,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"2be1b699-1ceb-4df5-a688-721b4e5c1884","name":"e2e-legacy-migration-fcd350da"} {"level":"error","ts":1789473192.6887784,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"871a0b33-7131-4e47-9f27-eafafd5d11ce","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473192.6888971,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"871a0b33-7131-4e47-9f27-eafafd5d11ce","error":"ExternalModel \"e2e-legacy-migration-fcd350da\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-fcd350da\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.6889613,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"bf8afaaf-17a7-4a7b-aa20-faa802e50aea"} {"level":"info","ts":1789473192.6889863,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"bf8afaaf-17a7-4a7b-aa20-faa802e50aea","provider":"e2e-legacy-migration-fcd350da","phase":"Pending"} {"level":"info","ts":1789473192.688819,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"8dc5ae87-b47d-4396-81ad-635efe4ffd18","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.6891901,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"8dc5ae87-b47d-4396-81ad-635efe4ffd18","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"error","ts":1789473192.6953275,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"2be1b699-1ceb-4df5-a688-721b4e5c1884","error":"failed to apply ExternalModel: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473192.6956148,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"82acbc21-f97f-49d5-a9f3-c3bab28b3787","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473192.6957026,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"82acbc21-f97f-49d5-a9f3-c3bab28b3787","error":"failed to apply Service: services \"e2e-legacy-migration-fcd350da\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473192.6956275,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"bf8afaaf-17a7-4a7b-aa20-faa802e50aea","error":"ExternalModel \"e2e-legacy-migration-fcd350da\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-fcd350da\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.6957963,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6c03886c-a71c-45e6-9cb5-4c9474da9846"} {"level":"info","ts":1789473192.6959274,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6c03886c-a71c-45e6-9cb5-4c9474da9846","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.7059402,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6fc3672c-c9de-4070-b13f-777fa4b2cf6f"} {"level":"info","ts":1789473192.7059748,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6fc3672c-c9de-4070-b13f-777fa4b2cf6f","provider":"e2e-legacy-migration-fcd350da","phase":"Pending"} {"level":"info","ts":1789473192.7059891,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6dcebfa9-5781-4746-a7a4-de8cf4f47714","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.7094195,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6dcebfa9-5781-4746-a7a4-de8cf4f47714","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.7133517,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"eb7e347e-f0f6-4400-a4eb-346e1fb66d5a","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.713474,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"eb7e347e-f0f6-4400-a4eb-346e1fb66d5a","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789473192.713484,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"65a37052-6c4e-4bbc-a1e6-a3eb2ad890c9","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.7135837,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"65a37052-6c4e-4bbc-a1e6-a3eb2ad890c9","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"info","ts":1789473192.7141004,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6c03886c-a71c-45e6-9cb5-4c9474da9846","service":"e2e-legacy-migration-fcd350da","serviceEntry":"e2e-legacy-migration-fcd350da","destinationRule":"e2e-legacy-migration-fcd350da"} {"level":"error","ts":1789473192.715015,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6fc3672c-c9de-4070-b13f-777fa4b2cf6f","error":"ExternalModel \"e2e-legacy-migration-fcd350da\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-fcd350da\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.7150733,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"d790154d-fbc9-41ff-823c-699e2c778ac0"} {"level":"info","ts":1789473192.7151084,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"d790154d-fbc9-41ff-823c-699e2c778ac0","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.721638,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6dcebfa9-5781-4746-a7a4-de8cf4f47714","provider":"openai","model":"e2e-legacy-migration-fcd350da","targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789473192.7219145,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6c03886c-a71c-45e6-9cb5-4c9474da9846","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:97\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.721998,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"01aaf801-756c-4bd5-a45a-4a2355c4f230"} {"level":"info","ts":1789473192.7220445,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"01aaf801-756c-4bd5-a45a-4a2355c4f230","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.7443411,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"90d1a32b-94db-4d40-8452-2352591576f9","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.7443843,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"90d1a32b-94db-4d40-8452-2352591576f9","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"error","ts":1789473192.7614543,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"d790154d-fbc9-41ff-823c-699e2c778ac0","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473192.7615144,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"d790154d-fbc9-41ff-823c-699e2c778ac0","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-fcd350da\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.7615876,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"4a249f7e-b878-4675-a518-3ae27558419e"} {"level":"info","ts":1789473192.7616532,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"4a249f7e-b878-4675-a518-3ae27558419e","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.7752664,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"01aaf801-756c-4bd5-a45a-4a2355c4f230","service":"e2e-legacy-migration-fcd350da","serviceEntry":"e2e-legacy-migration-fcd350da","destinationRule":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.7929277,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"2d6a7ab9-e0c6-4001-ae97-d64446de1050","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.7929654,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"2d6a7ab9-e0c6-4001-ae97-d64446de1050","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"error","ts":1789473192.7966073,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"4a249f7e-b878-4675-a518-3ae27558419e","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473192.7966607,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"4a249f7e-b878-4675-a518-3ae27558419e","error":"failed to apply HTTPRoute: Operation cannot be fulfilled on httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.7967227,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"eed75fc4-15a3-4e16-a262-668b1161e680"} {"level":"info","ts":1789473192.7967796,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"eed75fc4-15a3-4e16-a262-668b1161e680","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.831027,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"d81ead49-4152-40e4-a721-11d34e5e5be4","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.831065,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"d81ead49-4152-40e4-a721-11d34e5e5be4","modelName":"e2e-legacy-migration-fcd350da","resolvedRefs":1} {"level":"info","ts":1789473192.840715,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"eed75fc4-15a3-4e16-a262-668b1161e680","httpRoute":"e2e-legacy-migration-fcd350da","providers":["e2e-legacy-migration-fcd350da"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789473192.8512104,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"eed75fc4-15a3-4e16-a262-668b1161e680","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473192.8513052,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"e322752c-71ba-41fa-8c3c-faef987f86cd"} {"level":"info","ts":1789473192.8513713,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"e322752c-71ba-41fa-8c3c-faef987f86cd","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.8854513,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"e322752c-71ba-41fa-8c3c-faef987f86cd","httpRoute":"e2e-legacy-migration-fcd350da","providers":["e2e-legacy-migration-fcd350da"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789473192.96954,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"cd281ecd-d81e-43a5-ac31-2aabcada204f","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473192.9742963,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"cd281ecd-d81e-43a5-ac31-2aabcada204f","name":"e2e-legacy-migration-fcd350da"} {"level":"info","ts":1789473192.9852536,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"cd281ecd-d81e-43a5-ac31-2aabcada204f","provider":"openai","model":"e2e-legacy-migration-fcd350da","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789473195.6112745,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"aed8203e-20b2-433e-98af-1aa7905b2644","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473195.6195474,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"fb45d6a8-0f92-4d80-bc92-fa24c1cc7d88","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473195.6196327,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"fb45d6a8-0f92-4d80-bc92-fa24c1cc7d88","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473195.6197166,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"5454ecbf-504b-4c25-b4fc-01e79ad6c7cb"} {"level":"info","ts":1789473195.619739,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"0aba368c-c0f2-47d6-bbc3-7b0f72365cb7","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"error","ts":1789473195.619856,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"0aba368c-c0f2-47d6-bbc3-7b0f72365cb7","provider":"e2e-legacy-migration-fcd350da","error":"ExternalProvider \"e2e-legacy-migration-fcd350da\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473195.6199808,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"0aba368c-c0f2-47d6-bbc3-7b0f72365cb7"} {"level":"info","ts":1789473195.6198187,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"297b75f6-ac66-4993-81b8-e5316f72bf9b"} {"level":"error","ts":1789473195.620085,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"297b75f6-ac66-4993-81b8-e5316f72bf9b","provider":"e2e-legacy-migration-fcd350da","error":"ExternalProvider.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473195.6232,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6d7773c9-cd7e-43b5-a772-8a8ae159d0d3","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"info","ts":1789473195.623297,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"6d7773c9-cd7e-43b5-a772-8a8ae159d0d3","name":"e2e-legacy-migration-fcd350da","namespace":"llm"} {"level":"error","ts":1789473195.625417,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"297b75f6-ac66-4993-81b8-e5316f72bf9b","error":"externalmodels.inference.opendatahub.io \"e2e-legacy-migration-fcd350da\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789473195.6254523,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"297b75f6-ac66-4993-81b8-e5316f72bf9b","error":"ExternalModel \"e2e-legacy-migration-fcd350da\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-fcd350da\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789473195.6254926,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-fcd350da","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-fcd350da","reconcileID":"24cf8812-8b20-44b4-b136-5eddd42e9475"} {"level":"info","ts":1789473196.043272,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473196.0433676,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473196.043382,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473196.0433948,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","EoS":true} {"level":"info","ts":1789473196.04342,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","header":"x-maas-group"} {"level":"info","ts":1789473196.0434306,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","header":"x-maas-username"} {"level":"info","ts":1789473196.0434353,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","header":"x-maas-subscription"} {"level":"info","ts":1789473196.04345,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789473196.04346,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","modelName":"e2e-external-model"} {"level":"info","ts":1789473196.0434709,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789473196.0434926,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789473196.0435092,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","authType":"apikey"} {"level":"info","ts":1789473196.0435338,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473197.4422202,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473197.4422498,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473197.4422674,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","EoS":false} {"level":"info","ts":1789473197.4422805,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473197.4423144,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b","EoS":true} {"level":"info","ts":1789473197.4423234,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a77d32ee0cf49989fcfe95fd55f9f8ca","span_id":"7c7a3508bcdabc76","x-request-id":"c34523e0-c446-4177-87bf-f8d7315b5f1b"} {"level":"info","ts":1789473197.5110373,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473197.5111535,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473197.5111678,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473197.5111878,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","EoS":true} {"level":"info","ts":1789473197.5112238,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","header":"x-maas-username"} {"level":"info","ts":1789473197.5112383,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","header":"x-maas-group"} {"level":"info","ts":1789473197.5112486,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","header":"x-maas-subscription"} {"level":"info","ts":1789473197.511271,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789473197.5112998,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473198.806891,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473198.8069186,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473198.8069363,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","EoS":false} {"level":"info","ts":1789473198.8069496,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473198.806974,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1","EoS":true} {"level":"info","ts":1789473198.8069808,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e71046fbf681d4d83033dbdfcf943c93","span_id":"f745dfecc50d744d","x-request-id":"c9a85dd1-24ec-406e-864c-331365dd7ad1"} {"level":"info","ts":1789473198.8464375,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473198.8465285,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"info","ts":1789473198.8465412,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"info","ts":1789473198.8465524,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412","EoS":true} {"level":"info","ts":1789473198.8465824,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412","header":"x-maas-username"} {"level":"info","ts":1789473198.8465962,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412","header":"x-maas-subscription"} {"level":"info","ts":1789473198.846603,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412","header":"x-maas-group"} {"level":"info","ts":1789473198.8466322,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"info","ts":1789473199.763698,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"info","ts":1789473199.7637305,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"info","ts":1789473199.763752,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412","EoS":false} {"level":"info","ts":1789473199.7637725,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"info","ts":1789473199.7638187,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412","EoS":true} {"level":"info","ts":1789473199.7638323,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6108c1af00a263646613a7d37be89220","span_id":"16ee8dbf0b312d80","x-request-id":"43b2dc51-7d5e-44e5-a6d5-6ca067eb5412"} {"level":"error","ts":1789473199.7793188,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789473200.6563208,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"6642308f-67f9-42e9-82f2-f16a427a087a","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789473200.656409,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"6642308f-67f9-42e9-82f2-f16a427a087a","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789473200.6565154,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"5ee507d2-0041-4515-bf7d-122b2c224097"} {"level":"info","ts":1789473200.8130736,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"c79951be-36e9-4e86-8a74-e3b9f75752c9","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789473200.8131099,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"c79951be-36e9-4e86-8a74-e3b9f75752c9","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789473200.8130805,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"780bf680-8e0e-491a-aefd-b90e75a5a247"} {"level":"info","ts":1789473200.933936,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"b22e03b3-6e69-4da7-8ef5-8b750659e4cf","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789473200.9340196,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"b22e03b3-6e69-4da7-8ef5-8b750659e4cf","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789473204.3309019,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473204.3310003,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"df0c55faf414473fc8aa9468742fb8f2","span_id":"fde0ca677f54e57a","x-request-id":"4917743f-9f95-44b6-ad9f-6e38cf0aac5d"} {"level":"info","ts":1789473204.331014,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"df0c55faf414473fc8aa9468742fb8f2","span_id":"fde0ca677f54e57a","x-request-id":"4917743f-9f95-44b6-ad9f-6e38cf0aac5d"} {"level":"info","ts":1789473204.331026,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"df0c55faf414473fc8aa9468742fb8f2","span_id":"fde0ca677f54e57a","x-request-id":"4917743f-9f95-44b6-ad9f-6e38cf0aac5d","EoS":true} {"level":"info","ts":1789473204.3310773,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"df0c55faf414473fc8aa9468742fb8f2","span_id":"fde0ca677f54e57a","x-request-id":"4917743f-9f95-44b6-ad9f-6e38cf0aac5d"} {"level":"info","ts":1789473204.5033963,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473204.503509,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"eca047168b47993f40c9634cc46c8726","span_id":"f4bd34b0fa51b018","x-request-id":"a052494b-7242-44ba-be7a-932146a8db4f"} {"level":"info","ts":1789473204.503524,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"eca047168b47993f40c9634cc46c8726","span_id":"f4bd34b0fa51b018","x-request-id":"a052494b-7242-44ba-be7a-932146a8db4f"} {"level":"info","ts":1789473204.5035405,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"eca047168b47993f40c9634cc46c8726","span_id":"f4bd34b0fa51b018","x-request-id":"a052494b-7242-44ba-be7a-932146a8db4f","EoS":true} {"level":"info","ts":1789473204.5035877,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"eca047168b47993f40c9634cc46c8726","span_id":"f4bd34b0fa51b018","x-request-id":"a052494b-7242-44ba-be7a-932146a8db4f"} {"level":"info","ts":1789473204.6650634,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789473204.6651976,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"309811693ca2a4d65572b57c3bdb50cc","span_id":"3a4b9a88b9b20462","x-request-id":"cafb0bfd-4d0c-4f1a-b4b3-02315676c264"} {"level":"info","ts":1789473204.665214,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"309811693ca2a4d65572b57c3bdb50cc","span_id":"3a4b9a88b9b20462","x-request-id":"cafb0bfd-4d0c-4f1a-b4b3-02315676c264"} {"level":"info","ts":1789473204.6652327,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"309811693ca2a4d65572b57c3bdb50cc","span_id":"3a4b9a88b9b20462","x-request-id":"cafb0bfd-4d0c-4f1a-b4b3-02315676c264","EoS":true} {"level":"info","ts":1789473204.6652875,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"309811693ca2a4d65572b57c3bdb50cc","span_id":"3a4b9a88b9b20462","x-request-id":"cafb0bfd-4d0c-4f1a-b4b3-02315676c264"} {"level":"error","ts":1789473209.7799625,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789473219.7805305,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789473229.7814229,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474393.0668387,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474393.0669599,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.0669737,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.066986,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","EoS":true} {"level":"info","ts":1789474393.0670137,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","header":"x-maas-group"} {"level":"info","ts":1789474393.0670238,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","header":"x-maas-username"} {"level":"info","ts":1789474393.06703,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","header":"x-maas-subscription"} {"level":"info","ts":1789474393.0670452,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789474393.0670717,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.0681338,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.0681553,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.068172,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","EoS":false} {"level":"info","ts":1789474393.068183,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.068207,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c","EoS":true} {"level":"info","ts":1789474393.0682163,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5dfc0a069cac6f135a50aef40275f647","span_id":"518d8ec255c3dbc3","x-request-id":"25d4db25-84b3-4d48-a5c0-22203231810c"} {"level":"info","ts":1789474393.1087687,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474393.1088626,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"info","ts":1789474393.1088948,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"info","ts":1789474393.1089067,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","EoS":true} {"level":"info","ts":1789474393.1089294,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","header":"x-maas-username"} {"level":"info","ts":1789474393.1089392,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","header":"x-maas-group"} {"level":"info","ts":1789474393.108944,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","header":"x-maas-subscription"} {"level":"info","ts":1789474393.1089582,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789474393.1089818,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"info","ts":1789474393.1099613,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"info","ts":1789474393.1099849,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"info","ts":1789474393.1099982,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","EoS":false} {"level":"info","ts":1789474393.1100087,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"info","ts":1789474393.1100326,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4","EoS":true} {"level":"info","ts":1789474393.110041,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ef5fbf1d103cc21fa44c9d6030a66ad5","span_id":"08aa5438ac9ac61b","x-request-id":"63da473a-483b-45de-a720-be8e466c7ae4"} {"level":"error","ts":1789474404.914012,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474739.3731084,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474739.3732202,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.3732321,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.373246,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","EoS":true} {"level":"info","ts":1789474739.3732693,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","header":"x-maas-subscription"} {"level":"info","ts":1789474739.3732793,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","header":"x-maas-username"} {"level":"info","ts":1789474739.373288,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","header":"x-maas-group"} {"level":"info","ts":1789474739.3733063,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474739.373334,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.3857017,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.3857327,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.3857505,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","EoS":false} {"level":"info","ts":1789474739.3857636,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.3857884,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136","EoS":true} {"level":"info","ts":1789474739.3857982,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6a2b38bc4d20123d1e7aa59a26076087","span_id":"768b13438ad2727c","x-request-id":"9ec664d5-9f0d-4df6-9584-3b345e863136"} {"level":"info","ts":1789474739.5233357,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474739.5234292,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.523441,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.5234525,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","EoS":true} {"level":"info","ts":1789474739.5234742,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","header":"x-maas-username"} {"level":"info","ts":1789474739.5234842,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","header":"x-maas-subscription"} {"level":"info","ts":1789474739.523489,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","header":"x-maas-group"} {"level":"info","ts":1789474739.5235038,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474739.5235283,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.5243664,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.5243886,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.5244036,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","EoS":false} {"level":"info","ts":1789474739.5244176,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.5244424,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48","EoS":true} {"level":"info","ts":1789474739.5244536,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"923f637f0179a35b6a34965ba0c2bd84","span_id":"fd8acde9db1c84a8","x-request-id":"7c07e461-306b-46a8-ac76-db153a262f48"} {"level":"info","ts":1789474739.6597917,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474739.6598983,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.6599102,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.6599216,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","EoS":true} {"level":"info","ts":1789474739.6599438,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","header":"x-maas-username"} {"level":"info","ts":1789474739.659954,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","header":"x-maas-group"} {"level":"info","ts":1789474739.6599593,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","header":"x-maas-subscription"} {"level":"info","ts":1789474739.6599743,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474739.6599984,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.660858,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.6608999,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.6609237,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","EoS":false} {"level":"info","ts":1789474739.6609383,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.6609697,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7","EoS":true} {"level":"info","ts":1789474739.6609836,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2c7e1c71c44728e376fe08c4644c5d58","span_id":"46f67075b86e2918","x-request-id":"1ad9dbb8-ca81-4c74-b5fb-0513b8a2cdb7"} {"level":"info","ts":1789474739.7970865,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474739.7971892,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.7972043,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.7972167,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","EoS":true} {"level":"info","ts":1789474739.797244,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","header":"x-maas-username"} {"level":"info","ts":1789474739.7972546,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","header":"x-maas-subscription"} {"level":"info","ts":1789474739.7972608,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","header":"x-maas-group"} {"level":"info","ts":1789474739.7972765,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474739.797306,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.7991033,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.799123,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.7991426,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","EoS":false} {"level":"info","ts":1789474739.7991612,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.799189,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93","EoS":true} {"level":"info","ts":1789474739.7991984,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"23c75081004a6bd8f5d7212cf8b5642a","span_id":"cca24a7d27670228","x-request-id":"4554f303-ac0f-4a18-9108-d4e77b233d93"} {"level":"info","ts":1789474739.938321,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474739.9384522,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474739.9384685,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474739.9384828,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","EoS":true} {"level":"info","ts":1789474739.938508,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","header":"x-maas-group"} {"level":"info","ts":1789474739.9385185,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","header":"x-maas-subscription"} {"level":"info","ts":1789474739.938523,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","header":"x-maas-username"} {"level":"info","ts":1789474739.9385397,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474739.9385746,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474739.9394233,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474739.9394422,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474739.9394588,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","EoS":false} {"level":"info","ts":1789474739.9394734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474739.9395075,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4","EoS":true} {"level":"info","ts":1789474739.9395196,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3237ec4c0f5083d35f104be93532090c","span_id":"39c683faed6d2385","x-request-id":"6b169910-f223-4017-a284-b9fe8588d7b4"} {"level":"info","ts":1789474748.74717,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474748.7472756,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.7472882,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.7473001,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","EoS":true} {"level":"info","ts":1789474748.7473242,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","header":"x-maas-subscription"} {"level":"info","ts":1789474748.747334,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","header":"x-maas-group"} {"level":"info","ts":1789474748.74734,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","header":"x-maas-username"} {"level":"info","ts":1789474748.7473557,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474748.7473817,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.7482824,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.7483037,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.7483218,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","EoS":false} {"level":"info","ts":1789474748.748335,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.748362,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd","EoS":true} {"level":"info","ts":1789474748.7483714,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01ee7501798d180b16ef35a660d58b6b","span_id":"297bf13bc2abd530","x-request-id":"e3e8684d-2886-450f-a2e4-56c5fc4853fd"} {"level":"info","ts":1789474748.8584905,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474748.858612,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706"} {"level":"info","ts":1789474748.8594565,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706"} {"level":"info","ts":1789474748.8594813,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706"} {"level":"info","ts":1789474748.8594952,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706","EoS":false} {"level":"info","ts":1789474748.859508,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706"} {"level":"info","ts":1789474748.8595362,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706","EoS":true} {"level":"info","ts":1789474748.8595421,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"98981431cd483c090754eabbe5d53b39","span_id":"90cf9a3f0fe73952","x-request-id":"3696d5a7-0d02-4763-844f-2842ba3d6706"} {"level":"info","ts":1789474749.633643,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474749.63374,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"info","ts":1789474749.6337533,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"info","ts":1789474749.6337643,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","EoS":true} {"level":"info","ts":1789474749.6337874,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","header":"x-maas-group"} {"level":"info","ts":1789474749.633798,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","header":"x-maas-username"} {"level":"info","ts":1789474749.633803,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","header":"x-maas-subscription"} {"level":"info","ts":1789474749.6338181,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474749.6338432,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"info","ts":1789474749.6348124,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"info","ts":1789474749.634824,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"info","ts":1789474749.6348355,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","EoS":false} {"level":"info","ts":1789474749.6348474,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"info","ts":1789474749.634868,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c","EoS":true} {"level":"info","ts":1789474749.6348898,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8eb196031c4d408718f8d845eb514f32","span_id":"3bd11915d10fec6a","x-request-id":"07164c20-9db5-42b8-a948-9e925e7b582c"} {"level":"error","ts":1789474749.955095,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474750.5731897,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474750.5733056,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474750.5733225,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474750.5733354,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","EoS":true} {"level":"info","ts":1789474750.5733619,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","header":"x-maas-subscription"} {"level":"info","ts":1789474750.5733714,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","header":"x-maas-group"} {"level":"info","ts":1789474750.5733771,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","header":"x-maas-username"} {"level":"info","ts":1789474750.5733964,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474750.5734212,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474750.5744414,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474750.5744631,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474750.5744784,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","EoS":false} {"level":"info","ts":1789474750.5744908,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474750.5745242,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202","EoS":true} {"level":"info","ts":1789474750.5745392,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1c8c677a3164af07e94f15da07c98cd5","span_id":"207d10e29326cb88","x-request-id":"0f44a075-be74-422c-aaf3-db380006f202"} {"level":"info","ts":1789474751.3164005,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474751.3165123,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474751.3165255,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474751.316538,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","EoS":true} {"level":"info","ts":1789474751.3165617,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","header":"x-maas-username"} {"level":"info","ts":1789474751.3165727,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","header":"x-maas-group"} {"level":"info","ts":1789474751.3165781,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","header":"x-maas-subscription"} {"level":"info","ts":1789474751.3165996,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474751.316627,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474751.3175657,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474751.3175867,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474751.3176048,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","EoS":false} {"level":"info","ts":1789474751.3176186,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474751.3176408,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3","EoS":true} {"level":"info","ts":1789474751.3176498,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e98c91478779abccc8575808edbd7df8","span_id":"b59fdc9aa0f0b5b1","x-request-id":"c1cde80a-a6b3-4544-9c07-54ebd5f506a3"} {"level":"info","ts":1789474752.432595,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474752.432704,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"info","ts":1789474752.4327185,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"info","ts":1789474752.4327312,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","EoS":true} {"level":"info","ts":1789474752.4327564,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","header":"x-maas-group"} {"level":"info","ts":1789474752.432767,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","header":"x-maas-username"} {"level":"info","ts":1789474752.4327722,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","header":"x-maas-subscription"} {"level":"info","ts":1789474752.4327881,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474752.4328082,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"info","ts":1789474752.4339216,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"info","ts":1789474752.433943,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"info","ts":1789474752.4339595,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","EoS":false} {"level":"info","ts":1789474752.4339752,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"info","ts":1789474752.4340012,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7","EoS":true} {"level":"info","ts":1789474752.434011,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9e0fed6aa61018f5d368925c6a43e9a1","span_id":"4c47f857b2b913d6","x-request-id":"c3705e76-73e4-4d74-9372-7d92852cb4c7"} {"level":"error","ts":1789474764.9564352,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474772.217153,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474772.2172546,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"info","ts":1789474772.2172682,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"info","ts":1789474772.2172792,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","EoS":true} {"level":"info","ts":1789474772.217304,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","header":"x-maas-subscription"} {"level":"info","ts":1789474772.2173138,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","header":"x-maas-username"} {"level":"info","ts":1789474772.217319,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","header":"x-maas-group"} {"level":"info","ts":1789474772.2173343,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474772.21736,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"info","ts":1789474772.2184706,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"info","ts":1789474772.2184923,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"info","ts":1789474772.218507,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","EoS":false} {"level":"info","ts":1789474772.2185185,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"info","ts":1789474772.2185423,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791","EoS":true} {"level":"info","ts":1789474772.2185538,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a9ff3a376a4432303152a5d56bd609e7","span_id":"7a03412c4b2bae96","x-request-id":"7d21510d-fcff-42a1-9b12-53f43835f791"} {"level":"error","ts":1789474784.9584017,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474861.0893784,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474861.0894833,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"info","ts":1789474861.0894964,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"info","ts":1789474861.0895076,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","EoS":true} {"level":"info","ts":1789474861.0895305,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","header":"x-maas-group"} {"level":"info","ts":1789474861.0895398,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","header":"x-maas-subscription"} {"level":"info","ts":1789474861.0895445,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","header":"x-maas-username"} {"level":"info","ts":1789474861.089559,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789474861.0895846,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"info","ts":1789474861.1019225,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"info","ts":1789474861.101942,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"info","ts":1789474861.1019566,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","EoS":false} {"level":"info","ts":1789474861.1019685,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"info","ts":1789474861.1019905,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1","EoS":true} {"level":"info","ts":1789474861.1019995,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c6289204295f17bf06c5de946bdbd0a1","span_id":"8ee3c6f5a9d9451a","x-request-id":"dc8776fe-0cc6-4162-8845-af6dc26572e1"} {"level":"error","ts":1789474874.9708056,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474954.834339,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474954.8344498,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"info","ts":1789474954.8344631,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"info","ts":1789474954.8344753,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","EoS":true} {"level":"info","ts":1789474954.8345013,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","header":"x-maas-group"} {"level":"info","ts":1789474954.834516,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","header":"x-maas-subscription"} {"level":"info","ts":1789474954.834522,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","header":"x-maas-username"} {"level":"info","ts":1789474954.8345377,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789474954.8345609,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"info","ts":1789474954.836198,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"info","ts":1789474954.8362207,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"info","ts":1789474954.836236,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","EoS":false} {"level":"info","ts":1789474954.8362482,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"info","ts":1789474954.8362777,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206","EoS":true} {"level":"info","ts":1789474954.8362908,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"688430a6b6e0b0a28811d6edcca892f4","span_id":"b7cf4b0a0fc31855","x-request-id":"e8fedc28-b77e-4903-8117-c1039a89f206"} {"level":"error","ts":1789474964.9801023,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474968.8051548,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474968.805256,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.8052726,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.8052876,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","EoS":true} {"level":"info","ts":1789474968.8053076,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","header":"x-maas-group"} {"level":"info","ts":1789474968.8053184,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","header":"x-maas-username"} {"level":"info","ts":1789474968.8053243,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","header":"x-maas-subscription"} {"level":"info","ts":1789474968.8053389,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789474968.8053696,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.8187284,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.8187501,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.8187675,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","EoS":false} {"level":"info","ts":1789474968.8187828,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.8188133,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a","EoS":true} {"level":"info","ts":1789474968.818828,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c5d8fa4203d2af12ef1c5b0f95de3839","span_id":"2e991eb43a4acc0e","x-request-id":"d42418f4-9ef3-4585-8af2-ced31de30e8a"} {"level":"info","ts":1789474968.9568279,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474968.956953,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.9569669,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.956979,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","EoS":true} {"level":"info","ts":1789474968.9570012,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","header":"x-maas-username"} {"level":"info","ts":1789474968.957016,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","header":"x-maas-subscription"} {"level":"info","ts":1789474968.9570215,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","header":"x-maas-group"} {"level":"info","ts":1789474968.9570348,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789474968.957059,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.958078,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.9580977,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.958113,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","EoS":false} {"level":"info","ts":1789474968.9581256,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.9581575,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","EoS":false} {"level":"info","ts":1789474968.9581718,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474968.958185,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477","EoS":true} {"level":"info","ts":1789474968.958192,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"501900db03dd7e029bf200e9607337d4","span_id":"9f034c5cc892801d","x-request-id":"e8d09421-9803-41a1-96a0-3005d79f8477"} {"level":"info","ts":1789474969.0940037,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474969.0941105,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.0941231,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.0941346,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","EoS":true} {"level":"info","ts":1789474969.0941565,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","header":"x-maas-subscription"} {"level":"info","ts":1789474969.094166,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","header":"x-maas-username"} {"level":"info","ts":1789474969.094171,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","header":"x-maas-group"} {"level":"info","ts":1789474969.0941865,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789474969.0942125,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.0952692,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.0952904,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.0953054,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","EoS":false} {"level":"info","ts":1789474969.0953186,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.0953424,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","EoS":false} {"level":"info","ts":1789474969.0953524,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"info","ts":1789474969.095363,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b","EoS":true} {"level":"info","ts":1789474969.0953696,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2e6021ebea7624cebdd72d708808cc52","span_id":"fcb86fda1ce796ee","x-request-id":"bc755b2d-d8e2-43b9-a15d-4ec668db870b"} {"level":"error","ts":1789474979.9816904,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789474985.0595076,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789474985.0596092,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.0596213,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.0596333,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","EoS":true} {"level":"info","ts":1789474985.0596564,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","header":"x-maas-username"} {"level":"info","ts":1789474985.0596662,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","header":"x-maas-subscription"} {"level":"info","ts":1789474985.0596719,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","header":"x-maas-group"} {"level":"info","ts":1789474985.0596857,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789474985.0597117,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.0608323,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.0608509,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.0608845,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","EoS":false} {"level":"info","ts":1789474985.0609002,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.06093,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","EoS":false} {"level":"info","ts":1789474985.0609403,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"} {"level":"info","ts":1789474985.0609515,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393","EoS":true} {"level":"info","ts":1789474985.060961,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d24955f58e2199d36e918d8fc46c360c","span_id":"f73e1fddb2947187","x-request-id":"04451c8d-dba0-47c8-bfce-098851470393"}