{"level":"info","ts":1789391549.7408483,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"996bf79835cd1758ccd87698365fa70c","span_id":"b1d3e6ebba4554c8","x-request-id":"53bbc527-d08b-431c-a05e-2bbcecb2ff1f"} {"level":"error","ts":1789391552.1755133,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789391552.704979,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391552.7050774,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.705091,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.7051027,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","EoS":true} {"level":"info","ts":1789391552.705126,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","header":"x-maas-username"} {"level":"info","ts":1789391552.7051363,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","header":"x-maas-group"} {"level":"info","ts":1789391552.7051413,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","header":"x-maas-subscription"} {"level":"info","ts":1789391552.705157,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789391552.705185,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.7070684,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.707097,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.7071173,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","EoS":false} {"level":"info","ts":1789391552.7071342,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.7071674,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae","EoS":true} {"level":"info","ts":1789391552.7071824,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"396fd10a0b9f45eff3f57363ef67154e","span_id":"ab409369b14f7e2c","x-request-id":"1d5b9603-7ff2-4de2-a0b1-a1b7a44623ae"} {"level":"info","ts":1789391552.7554438,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391552.755547,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.7555594,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.7555711,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","EoS":true} {"level":"info","ts":1789391552.7555947,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","header":"x-maas-subscription"} {"level":"info","ts":1789391552.7556055,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","header":"x-maas-username"} {"level":"info","ts":1789391552.7556107,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","header":"x-maas-group"} {"level":"info","ts":1789391552.7556257,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789391552.7556567,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.7567203,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.756745,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.7567937,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","EoS":false} {"level":"info","ts":1789391552.7568116,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.756845,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb","EoS":true} {"level":"info","ts":1789391552.756858,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6d35231155658ca3e6d763bae5ae0482","span_id":"abf73568f3b51bfb","x-request-id":"2793ca33-6e18-4496-99dc-9da16b424edb"} {"level":"info","ts":1789391552.8000221,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391552.800162,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.8001795,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.8001976,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","EoS":true} {"level":"info","ts":1789391552.8002322,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","header":"x-maas-username"} {"level":"info","ts":1789391552.8002465,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","header":"x-maas-subscription"} {"level":"info","ts":1789391552.8002553,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","header":"x-maas-group"} {"level":"info","ts":1789391552.8002748,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789391552.8003094,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.8014534,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.8014793,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.801495,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","EoS":false} {"level":"info","ts":1789391552.8015094,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.801541,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d","EoS":true} {"level":"info","ts":1789391552.8015516,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"767d0e83aa0b7aabb8178262ad367f1d","span_id":"81f185f6f843b3f1","x-request-id":"1996fdea-6ce3-4108-b73c-5a4708bc0a2d"} {"level":"info","ts":1789391552.844654,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391552.844831,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391552.8448517,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391552.844869,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","EoS":true} {"level":"info","ts":1789391552.8448982,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","header":"x-maas-subscription"} {"level":"info","ts":1789391552.8449237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","header":"x-maas-username"} {"level":"info","ts":1789391552.8449292,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","header":"x-maas-group"} {"level":"info","ts":1789391552.8449457,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789391552.8449762,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391552.845931,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391552.8459516,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391552.8459666,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","EoS":false} {"level":"info","ts":1789391552.8459787,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391552.8460047,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc","EoS":true} {"level":"info","ts":1789391552.8460147,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"80d207af98d4f3a9e8f4d0abd0fafa42","span_id":"e27a7b4b65db5bd7","x-request-id":"cd3d2911-34e6-4573-b1c6-fc65a93de9bc"} {"level":"info","ts":1789391554.0375447,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"263c8555-c1fd-42d6-b506-2eedc7465638","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789391554.0376658,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"263c8555-c1fd-42d6-b506-2eedc7465638","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1789391554.0376148,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"c431eff7-b33a-4aaa-9ea1-f38ca7abf9a7"} {"level":"info","ts":1789391558.3859391,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391558.386136,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2ddf0f51abd6df65b2bf09d77467e9a2","span_id":"cfe18a5c9348a4c4","x-request-id":"adaba821-7903-4303-bfde-369c266c0bb0"} {"level":"info","ts":1789391558.3861713,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2ddf0f51abd6df65b2bf09d77467e9a2","span_id":"cfe18a5c9348a4c4","x-request-id":"adaba821-7903-4303-bfde-369c266c0bb0"} {"level":"info","ts":1789391558.3862004,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2ddf0f51abd6df65b2bf09d77467e9a2","span_id":"cfe18a5c9348a4c4","x-request-id":"adaba821-7903-4303-bfde-369c266c0bb0","EoS":true} {"level":"info","ts":1789391558.386251,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2ddf0f51abd6df65b2bf09d77467e9a2","span_id":"cfe18a5c9348a4c4","x-request-id":"adaba821-7903-4303-bfde-369c266c0bb0","path":"/llm/nonexistent-model-xyz/v1/chat/completions"} {"level":"info","ts":1789391558.386309,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2ddf0f51abd6df65b2bf09d77467e9a2","span_id":"cfe18a5c9348a4c4","x-request-id":"adaba821-7903-4303-bfde-369c266c0bb0"} {"level":"info","ts":1789391558.809118,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"5dca9aec-cc3e-4a37-9d53-5f302b4f86f8","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8165486,"caller":"legacymigration/reconciler.go:147","msg":"creating ExternalProvider","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"5dca9aec-cc3e-4a37-9d53-5f302b4f86f8","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.8303988,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"af4d7ca8-fc01-450b-93fc-87e8f789c299"} {"level":"info","ts":1789391558.8310974,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"399fde36-88f9-4e94-9114-aae5bcf41b32","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8311534,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"399fde36-88f9-4e94-9114-aae5bcf41b32","provider":"openai","endpoint":"httpbingo.org"} {"level":"error","ts":1789391558.8341348,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"5dca9aec-cc3e-4a37-9d53-5f302b4f86f8","error":"failed to apply ExternalProvider: externalproviders.inference.opendatahub.io \"e2e-legacy-migration-11115af3\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.8512776,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"fa90c650-c5d1-44ab-bb7d-ffc3ba00ea18","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8513315,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"fa90c650-c5d1-44ab-bb7d-ffc3ba00ea18","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789391558.85157,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"17769221-1b76-4cd7-ac98-63cd5c0ba4db","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8541045,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"484e3bbd-98f8-4313-8567-227f31d203da","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8541362,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"484e3bbd-98f8-4313-8567-227f31d203da","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"error","ts":1789391558.8543031,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"af4d7ca8-fc01-450b-93fc-87e8f789c299","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:84\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.8543615,"caller":"externalprovider/reconciler.go:187","msg":"Creating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"af4d7ca8-fc01-450b-93fc-87e8f789c299","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.8546176,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"df91891d-048a-46e5-be0f-7eabef967f78"} {"level":"info","ts":1789391558.8641937,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"17769221-1b76-4cd7-ac98-63cd5c0ba4db","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.870369,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"1102c02a-8ab7-4d87-9729-0aed60aa9276","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8704188,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"1102c02a-8ab7-4d87-9729-0aed60aa9276","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"error","ts":1789391558.8751488,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"df91891d-048a-46e5-be0f-7eabef967f78","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.875215,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"df91891d-048a-46e5-be0f-7eabef967f78","provider":"e2e-legacy-migration-11115af3","phase":"Pending"} {"level":"error","ts":1789391558.8759391,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"17769221-1b76-4cd7-ac98-63cd5c0ba4db","error":"failed to apply ExternalModel: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.8878875,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"52765f58-c57d-4dbb-9123-8a9d711215ff","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.8883913,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"f8544454-3430-4e47-b183-63bb67f27c32","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.88882,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"f8544454-3430-4e47-b183-63bb67f27c32","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"error","ts":1789391558.8884904,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"df91891d-048a-46e5-be0f-7eabef967f78","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789391558.8889866,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"df91891d-048a-46e5-be0f-7eabef967f78","error":"ExternalModel \"e2e-legacy-migration-11115af3\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-11115af3\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.8890975,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"ad048302-a801-4fca-a3dc-0d933920e9ea"} {"level":"info","ts":1789391558.8891792,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"ad048302-a801-4fca-a3dc-0d933920e9ea","provider":"e2e-legacy-migration-11115af3","phase":"Pending"} {"level":"error","ts":1789391558.8925805,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"af4d7ca8-fc01-450b-93fc-87e8f789c299","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:93\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789391558.8926325,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"af4d7ca8-fc01-450b-93fc-87e8f789c299","error":"failed to apply Service: services \"e2e-legacy-migration-11115af3\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.8926933,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"69f25436-9aac-4904-9526-ba0e9f69e195"} {"level":"info","ts":1789391558.8927794,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"69f25436-9aac-4904-9526-ba0e9f69e195","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.8959527,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"52765f58-c57d-4dbb-9123-8a9d711215ff","name":"e2e-legacy-migration-11115af3"} {"level":"error","ts":1789391558.901949,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"ad048302-a801-4fca-a3dc-0d933920e9ea","error":"ExternalModel \"e2e-legacy-migration-11115af3\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-11115af3\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.9063246,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"52765f58-c57d-4dbb-9123-8a9d711215ff","provider":"openai","model":"e2e-legacy-migration-11115af3","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789391558.9124029,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"06004651-2fcf-4832-b3f7-8f1b24e8412a"} {"level":"info","ts":1789391558.9124465,"caller":"externalmodel/reconciler.go:142","msg":"ExternalProvider not ready, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"06004651-2fcf-4832-b3f7-8f1b24e8412a","provider":"e2e-legacy-migration-11115af3","phase":"Pending"} {"level":"info","ts":1789391558.930315,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"079dcb8a-720d-46dd-b266-d21567971d29","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.9303663,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"079dcb8a-720d-46dd-b266-d21567971d29","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"error","ts":1789391558.9306083,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"06004651-2fcf-4832-b3f7-8f1b24e8412a","error":"ExternalModel \"e2e-legacy-migration-11115af3\": no provider refs resolved successfully: ExternalProvider \"e2e-legacy-migration-11115af3\" is not ready (phase: Pending)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.9307148,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"fcff593e-def9-4930-af20-9fcf4f732c4a","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.9307384,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"fcff593e-def9-4930-af20-9fcf4f732c4a","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1789391558.9308593,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"97a98712-2168-44dd-9f69-7fcb9cda2188"} {"level":"info","ts":1789391558.9309053,"caller":"externalmodel/reconciler.go:212","msg":"Creating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"97a98712-2168-44dd-9f69-7fcb9cda2188","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.9391584,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"69f25436-9aac-4904-9526-ba0e9f69e195","service":"e2e-legacy-migration-11115af3","serviceEntry":"e2e-legacy-migration-11115af3","destinationRule":"e2e-legacy-migration-11115af3"} {"level":"error","ts":1789391558.9680102,"caller":"externalprovider/reconciler.go:164","msg":"failed to update ExternalProvider status","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"69f25436-9aac-4904-9526-ba0e9f69e195","error":"Operation cannot be fulfilled on externalproviders.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:164\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalprovider/reconciler.go:97\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.968139,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"2b6a47c3-7e53-42fd-bc6f-180daefcf6a1"} {"level":"info","ts":1789391558.9682086,"caller":"externalprovider/reconciler.go:199","msg":"Updating Service","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"2b6a47c3-7e53-42fd-bc6f-180daefcf6a1","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.9700675,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"f1046a8f-4bfb-4ca7-93c6-b7c215d01500","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391558.97011,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"f1046a8f-4bfb-4ca7-93c6-b7c215d01500","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"error","ts":1789391558.9990418,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"97a98712-2168-44dd-9f69-7fcb9cda2188","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789391558.999106,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"97a98712-2168-44dd-9f69-7fcb9cda2188","error":"failed to apply HTTPRoute: httproutes.gateway.networking.k8s.io \"e2e-legacy-migration-11115af3\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391558.999188,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"4215adc6-9cfa-4a86-8150-eaf4d3c0e873"} {"level":"info","ts":1789391558.9992595,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"4215adc6-9cfa-4a86-8150-eaf4d3c0e873","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391558.9998899,"caller":"externalprovider/reconciler.go:131","msg":"ExternalProvider resources reconciled","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"2b6a47c3-7e53-42fd-bc6f-180daefcf6a1","service":"e2e-legacy-migration-11115af3","serviceEntry":"e2e-legacy-migration-11115af3","destinationRule":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391559.0169814,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"4ad800f8-f937-4974-91df-6eb1a153e618","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391559.0170286,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"4ad800f8-f937-4974-91df-6eb1a153e618","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"info","ts":1789391559.0336113,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"4215adc6-9cfa-4a86-8150-eaf4d3c0e873","httpRoute":"e2e-legacy-migration-11115af3","providers":["e2e-legacy-migration-11115af3"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789391559.0567243,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"4215adc6-9cfa-4a86-8150-eaf4d3c0e873","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391559.0570526,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"eb7ca9c8-c2a4-45d9-8f13-cecebc2d9a63"} {"level":"info","ts":1789391559.0571954,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"eb7ca9c8-c2a4-45d9-8f13-cecebc2d9a63","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391559.062281,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"8f8a12b8-924f-43a5-acb8-8b7ac8a99405","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391559.062361,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"8f8a12b8-924f-43a5-acb8-8b7ac8a99405","modelName":"e2e-legacy-migration-11115af3","resolvedRefs":1} {"level":"info","ts":1789391559.0867426,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"eb7ca9c8-c2a4-45d9-8f13-cecebc2d9a63","httpRoute":"e2e-legacy-migration-11115af3","providers":["e2e-legacy-migration-11115af3"],"targetModel":"gpt-3.5-turbo"} {"level":"error","ts":1789391559.106686,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"eb7ca9c8-c2a4-45d9-8f13-cecebc2d9a63","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-legacy-migration-11115af3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:112\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789391559.106918,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"47272c86-83c9-4c3f-8b23-61ae2ef28a20"} {"level":"info","ts":1789391559.1070452,"caller":"externalmodel/reconciler.go:226","msg":"Updating HTTPRoute","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"47272c86-83c9-4c3f-8b23-61ae2ef28a20","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391559.1743634,"caller":"externalmodel/reconciler.go:186","msg":"ExternalModel HTTPRoute reconciled","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"47272c86-83c9-4c3f-8b23-61ae2ef28a20","httpRoute":"e2e-legacy-migration-11115af3","providers":["e2e-legacy-migration-11115af3"],"targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789391559.3364024,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"e4efcb0f-211b-4142-8725-ad07881e343a","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391559.3643458,"caller":"legacymigration/reconciler.go:183","msg":"updating ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"e4efcb0f-211b-4142-8725-ad07881e343a","name":"e2e-legacy-migration-11115af3"} {"level":"info","ts":1789391559.3849628,"caller":"legacymigration/reconciler.go:138","msg":"legacy migration complete","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"e4efcb0f-211b-4142-8725-ad07881e343a","provider":"openai","model":"e2e-legacy-migration-11115af3","targetModel":"gpt-3.5-turbo"} {"level":"info","ts":1789391561.909406,"caller":"legacymigration/reconciler.go:61","msg":"reconciling legacy ExternalModel","controller":"legacy-migration","controllerGroup":"maas.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"e56ef065-bce6-438b-a00c-460f90d63380","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391561.9255807,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"8b86f7f7-f484-4b29-a812-bec51e2b1bbe","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391561.9257302,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"8b86f7f7-f484-4b29-a812-bec51e2b1bbe","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391561.9261527,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"57b4ad67-6478-441c-b033-76d0cf3c6071"} {"level":"info","ts":1789391561.9263408,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"6306b2a2-579a-4aa8-8765-ad22dac1c887"} {"level":"info","ts":1789391561.926436,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"7c605558-d789-4d83-ac95-073b006f511b","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"info","ts":1789391561.9264953,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-11115af3","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-11115af3","reconcileID":"7c605558-d789-4d83-ac95-073b006f511b","name":"e2e-legacy-migration-11115af3","namespace":"llm"} {"level":"error","ts":1789391562.176074,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789391562.3901217,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391562.3902555,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.3902707,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.390287,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","EoS":true} {"level":"info","ts":1789391562.3903122,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","header":"x-maas-subscription"} {"level":"info","ts":1789391562.390325,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","header":"x-maas-group"} {"level":"info","ts":1789391562.3903332,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","header":"x-maas-username"} {"level":"info","ts":1789391562.3903549,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789391562.3903692,"caller":"model-provider-resolver/plugin.go:212","msg":"resolved model by name","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","modelName":"e2e-external-model"} {"level":"info","ts":1789391562.3903875,"caller":"model-provider-resolver/plugin.go:264","msg":"external model resolved","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","model":"e2e-external-model","provider":"openai","inputFormat":"openai-chat","apiFormat":"openai-chat"} {"level":"info","ts":1789391562.3904233,"caller":"api-translation/plugin.go:169","msg":"request api-translation completed successfully","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","input":"openai-chat","output":"openai-chat"} {"level":"info","ts":1789391562.390453,"caller":"apikey-injection/plugin.go:200","msg":"auth headers injected","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","authType":"apikey"} {"level":"info","ts":1789391562.3904896,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.6487424,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.6488512,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.6488812,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","EoS":false} {"level":"info","ts":1789391562.648902,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.6489544,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3","EoS":true} {"level":"info","ts":1789391562.6489663,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f02c3eeee5674897159ded452705c83","span_id":"9f18b59b2cf303e6","x-request-id":"e57d3039-4d83-4eae-acb0-30350d4a75b3"} {"level":"info","ts":1789391562.6972942,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391562.6974168,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.697431,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.697449,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","EoS":true} {"level":"info","ts":1789391562.6974885,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","header":"x-maas-subscription"} {"level":"info","ts":1789391562.697502,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","header":"x-maas-group"} {"level":"info","ts":1789391562.69751,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","header":"x-maas-username"} {"level":"info","ts":1789391562.6975303,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","path":"/llm/e2e-external-model/v1/chat/completions"} {"level":"info","ts":1789391562.6975918,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.8870559,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.887328,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.88738,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","EoS":false} {"level":"info","ts":1789391562.8874025,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.887489,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4","EoS":true} {"level":"info","ts":1789391562.88753,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"53aafaffa905e221ffa60fef259b3aba","span_id":"dcd9e89e61bf866e","x-request-id":"576752c2-bc19-41a8-95a3-91c2285f51d4"} {"level":"info","ts":1789391562.936234,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391562.9363546,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391562.9363766,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391562.9363914,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f","EoS":true} {"level":"info","ts":1789391562.936415,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f","header":"x-maas-username"} {"level":"info","ts":1789391562.9364243,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f","header":"x-maas-group"} {"level":"info","ts":1789391562.936433,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f","header":"x-maas-subscription"} {"level":"info","ts":1789391562.9364722,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391563.1317897,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391563.1319103,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391563.1319609,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f","EoS":false} {"level":"info","ts":1789391563.1319833,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391563.1320338,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f","EoS":true} {"level":"info","ts":1789391563.1320472,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dd8ffd2a6d26c620378064a47581e59e","span_id":"76912d1690cc4e8a","x-request-id":"29e5f27e-b75d-4a9a-8cef-eb27925ddf6f"} {"level":"info","ts":1789391563.986068,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"4e461258-e1f4-48cd-a072-5784f321245d","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789391563.9861724,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"4e461258-e1f4-48cd-a072-5784f321245d","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789391563.9862611,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"436a3b99-6a68-45aa-bcfe-c1c05c90224a"} {"level":"info","ts":1789391564.1462371,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"d1a6810a-8df6-4b46-8098-71fa51e8fdcc"} {"level":"info","ts":1789391564.1463802,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"86235907-8085-4fda-bcc1-4d79dd0b6bfe","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789391564.146434,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"86235907-8085-4fda-bcc1-4d79dd0b6bfe","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789391564.2731118,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7f4bccb8-961c-45ba-8d5e-55c83b831eed","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789391564.273173,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7f4bccb8-961c-45ba-8d5e-55c83b831eed","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789391569.6095943,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391569.609732,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"96ac40893df52fa77b076af41b1a9df2","span_id":"a0853b79eeaf44c4","x-request-id":"d168bdb3-1de8-4839-bc24-75d3042cd561"} {"level":"info","ts":1789391569.6097689,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"96ac40893df52fa77b076af41b1a9df2","span_id":"a0853b79eeaf44c4","x-request-id":"d168bdb3-1de8-4839-bc24-75d3042cd561"} {"level":"info","ts":1789391569.6097875,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"96ac40893df52fa77b076af41b1a9df2","span_id":"a0853b79eeaf44c4","x-request-id":"d168bdb3-1de8-4839-bc24-75d3042cd561","EoS":true} {"level":"info","ts":1789391569.6098528,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"96ac40893df52fa77b076af41b1a9df2","span_id":"a0853b79eeaf44c4","x-request-id":"d168bdb3-1de8-4839-bc24-75d3042cd561"} {"level":"info","ts":1789391569.8023233,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391569.8024628,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c1e1c130264bc9c34859d2e254a7c104","span_id":"760a9a20fe52bdb1","x-request-id":"b086a89a-28c4-4e06-82f4-10c3fad1f7bb"} {"level":"info","ts":1789391569.8024807,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c1e1c130264bc9c34859d2e254a7c104","span_id":"760a9a20fe52bdb1","x-request-id":"b086a89a-28c4-4e06-82f4-10c3fad1f7bb"} {"level":"info","ts":1789391569.802499,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c1e1c130264bc9c34859d2e254a7c104","span_id":"760a9a20fe52bdb1","x-request-id":"b086a89a-28c4-4e06-82f4-10c3fad1f7bb","EoS":true} {"level":"info","ts":1789391569.8025544,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c1e1c130264bc9c34859d2e254a7c104","span_id":"760a9a20fe52bdb1","x-request-id":"b086a89a-28c4-4e06-82f4-10c3fad1f7bb"} {"level":"info","ts":1789391570.0019748,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789391570.0021064,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"69dcf7450fd2335bd6e2a7f5f279ddbe","span_id":"b73f0944e1c95d06","x-request-id":"79685ce7-2cc9-4ee6-95d9-9570785b3331"} {"level":"info","ts":1789391570.0021214,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"69dcf7450fd2335bd6e2a7f5f279ddbe","span_id":"b73f0944e1c95d06","x-request-id":"79685ce7-2cc9-4ee6-95d9-9570785b3331"} {"level":"info","ts":1789391570.0021377,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"69dcf7450fd2335bd6e2a7f5f279ddbe","span_id":"b73f0944e1c95d06","x-request-id":"79685ce7-2cc9-4ee6-95d9-9570785b3331","EoS":true} {"level":"info","ts":1789391570.0021946,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"69dcf7450fd2335bd6e2a7f5f279ddbe","span_id":"b73f0944e1c95d06","x-request-id":"79685ce7-2cc9-4ee6-95d9-9570785b3331"} {"level":"error","ts":1789391572.1770282,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789391587.1786194,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789392694.5198097,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392694.519915,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.5199275,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.5199397,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","EoS":true} {"level":"info","ts":1789392694.5199625,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","header":"x-maas-username"} {"level":"info","ts":1789392694.5199707,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","header":"x-maas-subscription"} {"level":"info","ts":1789392694.5199761,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","header":"x-maas-group"} {"level":"info","ts":1789392694.5199916,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392694.5200193,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.5333269,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.533356,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.5333743,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","EoS":false} {"level":"info","ts":1789392694.5333862,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.5334089,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038","EoS":true} {"level":"info","ts":1789392694.5334158,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c361e3e4232ea3506ba6a455651704fb","span_id":"c2cac7447d8b1b61","x-request-id":"40c38cc5-6e6a-4f8b-a63c-535b8a82a038"} {"level":"info","ts":1789392694.673455,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392694.6735787,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.6735952,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.6736073,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","EoS":true} {"level":"info","ts":1789392694.6736321,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","header":"x-maas-subscription"} {"level":"info","ts":1789392694.673643,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","header":"x-maas-username"} {"level":"info","ts":1789392694.6736484,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","header":"x-maas-group"} {"level":"info","ts":1789392694.673663,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392694.673691,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.6758106,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.6758304,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.6758466,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","EoS":false} {"level":"info","ts":1789392694.6758635,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.6758926,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f","EoS":true} {"level":"info","ts":1789392694.6759014,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fdfc166170b29bcc50bda075d4711f91","span_id":"0ee6f0330634907e","x-request-id":"ccc04bf4-1fa1-4474-8a0e-22a861509a9f"} {"level":"info","ts":1789392694.8114529,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392694.811549,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.8115609,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.8115726,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","EoS":true} {"level":"info","ts":1789392694.8115957,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","header":"x-maas-username"} {"level":"info","ts":1789392694.8116062,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","header":"x-maas-subscription"} {"level":"info","ts":1789392694.8116117,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","header":"x-maas-group"} {"level":"info","ts":1789392694.811626,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392694.8116531,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.8125045,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.8125248,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.8125374,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","EoS":false} {"level":"info","ts":1789392694.8125584,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.8125846,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704","EoS":true} {"level":"info","ts":1789392694.812594,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"18717a38eb61ca9cfa17b5b5a7350e16","span_id":"42ba3d693bd0313b","x-request-id":"2fd7d4cf-29d2-4ae2-af13-584656db9704"} {"level":"info","ts":1789392694.9521031,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392694.952197,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392694.9522092,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392694.9522204,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","EoS":true} {"level":"info","ts":1789392694.9522464,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","header":"x-maas-username"} {"level":"info","ts":1789392694.9522562,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","header":"x-maas-subscription"} {"level":"info","ts":1789392694.9522629,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","header":"x-maas-group"} {"level":"info","ts":1789392694.9522784,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392694.952308,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392694.9531853,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392694.9532053,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392694.9532175,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","EoS":false} {"level":"info","ts":1789392694.9532359,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392694.9532578,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17","EoS":true} {"level":"info","ts":1789392694.9532676,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"869b380ef81674d29386b75733cae9ed","span_id":"4e0f7f0134ef601b","x-request-id":"ba1518fb-0e90-4b24-b0ba-787d1f424d17"} {"level":"info","ts":1789392695.094191,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392695.0942934,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392695.0943048,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392695.0943167,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","EoS":true} {"level":"info","ts":1789392695.0943394,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","header":"x-maas-username"} {"level":"info","ts":1789392695.0943513,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","header":"x-maas-group"} {"level":"info","ts":1789392695.0943568,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","header":"x-maas-subscription"} {"level":"info","ts":1789392695.0943713,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392695.0943983,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392695.095428,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392695.0954473,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392695.0954604,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","EoS":false} {"level":"info","ts":1789392695.095478,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392695.0955045,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48","EoS":true} {"level":"info","ts":1789392695.0955122,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ca48aadf5c8795b36d8d6408e32cd53","span_id":"b7928e76d30336c2","x-request-id":"2d86e583-7818-4555-a215-ea6e4f93fc48"} {"level":"info","ts":1789392703.9014244,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392703.9015338,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392703.9015532,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392703.9015672,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","EoS":true} {"level":"info","ts":1789392703.9015915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","header":"x-maas-username"} {"level":"info","ts":1789392703.9016032,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","header":"x-maas-subscription"} {"level":"info","ts":1789392703.901609,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","header":"x-maas-group"} {"level":"info","ts":1789392703.9016263,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392703.9016607,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392703.9027386,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392703.9027905,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392703.9028113,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","EoS":false} {"level":"info","ts":1789392703.902831,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392703.9028702,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8","EoS":true} {"level":"info","ts":1789392703.9028814,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c7e82923c901a67b07257a98603b3aeb","span_id":"6d3304b3cb0477ac","x-request-id":"ce6070b5-cc95-48a9-b19e-73d15abf98d8"} {"level":"info","ts":1789392704.0188408,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392704.0189452,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298"} {"level":"info","ts":1789392704.0199094,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298"} {"level":"info","ts":1789392704.0199308,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298"} {"level":"info","ts":1789392704.0199473,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298","EoS":false} {"level":"info","ts":1789392704.0199645,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298"} {"level":"info","ts":1789392704.0200026,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298","EoS":true} {"level":"info","ts":1789392704.0200198,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3f857feae49656b077ff7b954bb7a8e1","span_id":"875b2006548d5916","x-request-id":"22d4cc79-a96b-4ed4-bd45-f50734e75298"} {"level":"info","ts":1789392705.8027897,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392705.802905,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"info","ts":1789392705.802921,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"info","ts":1789392705.8029335,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","EoS":true} {"level":"info","ts":1789392705.8029618,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","header":"x-maas-subscription"} {"level":"info","ts":1789392705.802976,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","header":"x-maas-group"} {"level":"info","ts":1789392705.8029835,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","header":"x-maas-username"} {"level":"info","ts":1789392705.8029995,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392705.8030264,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"info","ts":1789392705.803994,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"info","ts":1789392705.8040187,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"info","ts":1789392705.8040364,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","EoS":false} {"level":"info","ts":1789392705.8040626,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"info","ts":1789392705.8040936,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70","EoS":true} {"level":"info","ts":1789392705.8041017,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"df6f705bc9bc451e2f67f7852f2a0126","span_id":"c96a7acceb4ac8f9","x-request-id":"a6b344f6-f533-4428-80d2-43d47432fa70"} {"level":"error","ts":1789392707.3203013,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789392708.8464515,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392708.8465507,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392708.846564,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392708.8465753,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","EoS":true} {"level":"info","ts":1789392708.8466005,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","header":"x-maas-subscription"} {"level":"info","ts":1789392708.8466117,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","header":"x-maas-group"} {"level":"info","ts":1789392708.8466167,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","header":"x-maas-username"} {"level":"info","ts":1789392708.846632,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392708.8466573,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392708.847622,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392708.8476422,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392708.8476574,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","EoS":false} {"level":"info","ts":1789392708.8476686,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392708.847695,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67","EoS":true} {"level":"info","ts":1789392708.8477051,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f3d0ceb0703f05c68aae4752a246039a","span_id":"aef65bc46eaf5512","x-request-id":"8c1fd593-b8de-4893-8d4f-885ffe8f6d67"} {"level":"info","ts":1789392709.5566518,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392709.5568206,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392709.5568423,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392709.5568705,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","EoS":true} {"level":"info","ts":1789392709.556899,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","header":"x-maas-username"} {"level":"info","ts":1789392709.5569103,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","header":"x-maas-subscription"} {"level":"info","ts":1789392709.5569155,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","header":"x-maas-group"} {"level":"info","ts":1789392709.5569305,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392709.5569582,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392709.5580158,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392709.5580347,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392709.5580459,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","EoS":false} {"level":"info","ts":1789392709.558059,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392709.5580866,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b","EoS":true} {"level":"info","ts":1789392709.5580964,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6dfbcc1ba66a3af448ec324c7197cea0","span_id":"1863d85f4acb70d7","x-request-id":"acef4059-d1b9-4ffb-be9c-1fe43112056b"} {"level":"info","ts":1789392710.618281,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392710.6183865,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"info","ts":1789392710.6184053,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"info","ts":1789392710.6184187,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","EoS":true} {"level":"info","ts":1789392710.6184418,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","header":"x-maas-subscription"} {"level":"info","ts":1789392710.6184523,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","header":"x-maas-group"} {"level":"info","ts":1789392710.6184576,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","header":"x-maas-username"} {"level":"info","ts":1789392710.6184754,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392710.618508,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"info","ts":1789392710.6195028,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"info","ts":1789392710.619524,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"info","ts":1789392710.6195354,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","EoS":false} {"level":"info","ts":1789392710.6195564,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"info","ts":1789392710.6195872,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484","EoS":true} {"level":"info","ts":1789392710.6195953,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7e5d75ec6e02feb4ddcbca4a6858b2a2","span_id":"dddac04d78ed7ebc","x-request-id":"de3a3f8c-876a-44bd-9bd0-556efc418484"} {"level":"error","ts":1789392722.321135,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789392730.192298,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392730.1924198,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"info","ts":1789392730.192437,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"info","ts":1789392730.1924498,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","EoS":true} {"level":"info","ts":1789392730.1924725,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","header":"x-maas-group"} {"level":"info","ts":1789392730.1924837,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","header":"x-maas-username"} {"level":"info","ts":1789392730.1924887,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","header":"x-maas-subscription"} {"level":"info","ts":1789392730.192503,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392730.1925285,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"info","ts":1789392730.1934733,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"info","ts":1789392730.1934946,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"info","ts":1789392730.1935067,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","EoS":false} {"level":"info","ts":1789392730.1935203,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"info","ts":1789392730.1935406,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93","EoS":true} {"level":"info","ts":1789392730.1935503,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"39f1d0e854459542c3afb33e2017f6eb","span_id":"b58ad1667c1cc2e4","x-request-id":"0ed27ccb-d631-4caf-bd09-5c8958ce5b93"} {"level":"error","ts":1789392742.3231237,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789392817.7135048,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392817.713642,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"info","ts":1789392817.7136679,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"info","ts":1789392817.713687,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","EoS":true} {"level":"info","ts":1789392817.7137191,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","header":"x-maas-subscription"} {"level":"info","ts":1789392817.713735,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","header":"x-maas-group"} {"level":"info","ts":1789392817.7137432,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","header":"x-maas-username"} {"level":"info","ts":1789392817.7137797,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789392817.713809,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"info","ts":1789392817.7265935,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"info","ts":1789392817.726621,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"info","ts":1789392817.7266414,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","EoS":false} {"level":"info","ts":1789392817.7266583,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"info","ts":1789392817.7266927,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b","EoS":true} {"level":"info","ts":1789392817.7267067,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"08c63c49cec21efd0226d9dce8452cd5","span_id":"0c6b0412d52a1f30","x-request-id":"82d44806-8347-45c4-a86e-c0590bc2dd3b"} {"level":"error","ts":1789392832.332732,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789392869.9223824,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392869.922518,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"info","ts":1789392869.9225383,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"info","ts":1789392869.9225893,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","EoS":true} {"level":"info","ts":1789392869.9226296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","header":"x-maas-group"} {"level":"info","ts":1789392869.9226477,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","header":"x-maas-subscription"} {"level":"info","ts":1789392869.9226573,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","header":"x-maas-username"} {"level":"info","ts":1789392869.9226813,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789392869.9227252,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"info","ts":1789392869.924742,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"info","ts":1789392869.924788,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"info","ts":1789392869.9248066,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","EoS":false} {"level":"info","ts":1789392869.924819,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"info","ts":1789392869.9248896,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11","EoS":true} {"level":"info","ts":1789392869.9249027,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82028961242c8f27889b0b5377cc6e43","span_id":"8222092a06415a32","x-request-id":"f6d11abb-d888-4a38-8b46-c9567e0cbf11"} {"level":"error","ts":1789392882.3382487,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789392883.6560276,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392883.6561394,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.6561527,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.6561646,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","EoS":true} {"level":"info","ts":1789392883.656189,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","header":"x-maas-subscription"} {"level":"info","ts":1789392883.6561995,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","header":"x-maas-username"} {"level":"info","ts":1789392883.6562068,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","header":"x-maas-group"} {"level":"info","ts":1789392883.6562216,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789392883.6562486,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.669401,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.6694286,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.669444,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","EoS":false} {"level":"info","ts":1789392883.6694548,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.6694844,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","EoS":false} {"level":"info","ts":1789392883.669501,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.6695132,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594","EoS":true} {"level":"info","ts":1789392883.6695273,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"daeefb8416dd5d5caff93b8ab0e5f286","span_id":"bb645d8ff05b5045","x-request-id":"0120b45f-49a7-494d-b962-7c2ba2b6c594"} {"level":"info","ts":1789392883.812972,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392883.8130972,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8131135,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8131263,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","EoS":true} {"level":"info","ts":1789392883.813152,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","header":"x-maas-subscription"} {"level":"info","ts":1789392883.813163,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","header":"x-maas-group"} {"level":"info","ts":1789392883.8131683,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","header":"x-maas-username"} {"level":"info","ts":1789392883.8131843,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789392883.813212,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8153667,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8153808,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8153925,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","EoS":false} {"level":"info","ts":1789392883.8154023,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8154273,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","EoS":false} {"level":"info","ts":1789392883.8154414,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.8154535,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef","EoS":true} {"level":"info","ts":1789392883.8154612,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6f1567203d7420ec64ac79a084880c86","span_id":"bd46c305f4ec9a1e","x-request-id":"e9cc314f-4d2c-4596-8f25-f5046e1cb7ef"} {"level":"info","ts":1789392883.9522107,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392883.952333,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.95235,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.952364,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","EoS":true} {"level":"info","ts":1789392883.9523902,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","header":"x-maas-subscription"} {"level":"info","ts":1789392883.9524045,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","header":"x-maas-group"} {"level":"info","ts":1789392883.9524138,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","header":"x-maas-username"} {"level":"info","ts":1789392883.9524302,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789392883.9524627,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.9534938,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.9535196,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.9535406,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","EoS":false} {"level":"info","ts":1789392883.9535573,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.9535959,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","EoS":false} {"level":"info","ts":1789392883.9536157,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392883.9536316,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692","EoS":true} {"level":"info","ts":1789392883.9536436,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e15900859c6a7deee1f498d0a46197f8","span_id":"22fc8bb7d9cb66f9","x-request-id":"16687eb3-4856-4952-8774-9affbdb36692"} {"level":"info","ts":1789392896.7783773,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789392896.7784817,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.7784998,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.7785137,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","EoS":true} {"level":"info","ts":1789392896.7785344,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","header":"x-maas-subscription"} {"level":"info","ts":1789392896.7785454,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","header":"x-maas-username"} {"level":"info","ts":1789392896.77855,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","header":"x-maas-group"} {"level":"info","ts":1789392896.7785654,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789392896.7785993,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.7804072,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.780421,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.780435,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","EoS":false} {"level":"info","ts":1789392896.7804558,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.780482,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","EoS":false} {"level":"info","ts":1789392896.7804954,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"info","ts":1789392896.7805042,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff","EoS":true} {"level":"info","ts":1789392896.7805111,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"137e224e83f3f9d89f54ddf63ea34fdd","span_id":"ec5b52b63c5dc18e","x-request-id":"71995762-9654-4388-9128-6d69f3475bff"} {"level":"error","ts":1789392897.3394082,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789392907.34006,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}