{"level":"info","ts":1789669008.6067414,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fb5cbbf5a0c35f13ddc7d006b6852eb1","span_id":"16afddcb367181e8","x-request-id":"25b07ba6-38f4-43ea-961c-9fdc0faecc9f"} {"level":"info","ts":1789669008.6591043,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789669008.6592064,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669008.659221,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669008.6592395,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","EoS":true} {"level":"info","ts":1789669008.6592734,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","header":"x-maas-subscription"} {"level":"info","ts":1789669008.6592858,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","header":"x-maas-group"} {"level":"info","ts":1789669008.6592934,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","header":"x-maas-keyname"} {"level":"info","ts":1789669008.6593,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","header":"x-maas-username"} {"level":"info","ts":1789669008.6593218,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789669008.6593547,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669008.6602333,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669008.6602542,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669008.660275,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","EoS":false} {"level":"info","ts":1789669008.6602883,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669008.6603153,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0","EoS":true} {"level":"info","ts":1789669008.6603248,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d538644676815f38c22e39695ced8c0c","span_id":"a6ab98fc0c9349a1","x-request-id":"a3ad1468-e274-4e07-b59b-2e6c030420c0"} {"level":"info","ts":1789669009.3660104,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"65efccb2-4a08-4343-92e3-7fb33a709cf0","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789669009.3661366,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"65efccb2-4a08-4343-92e3-7fb33a709cf0","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1789669009.3660102,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"dbf5bb12-7ecf-4c74-ba3d-b96254593872"} {"level":"info","ts":1789669009.5003383,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"dcdf5853-1cb4-4bdf-a232-cdcecc982701","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789669009.5004506,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"dcdf5853-1cb4-4bdf-a232-cdcecc982701","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1789669009.500375,"caller":"externalprovider/reconciler.go:69","msg":"Reconciling ExternalProvider","controller":"external-provider-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"31eb25f7-cb97-4354-9eda-9814c2967328"} {"level":"info","ts":1789669009.6036,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"e04384e4-5221-4f82-8bfb-c15f14f986f2","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1789669009.6036422,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"e04384e4-5221-4f82-8bfb-c15f14f986f2","key":"llm/e2e-external-model-api-key"} {"level":"error","ts":1789669014.207087,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789669023.2801816,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789669023.2803278,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9ba8e55654aab812a6e71e04d31c3357","span_id":"200757c7cb97cd54","x-request-id":"01d51e54-4a86-4b9f-82cc-cbd453110b19"} {"level":"info","ts":1789669023.2803504,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9ba8e55654aab812a6e71e04d31c3357","span_id":"200757c7cb97cd54","x-request-id":"01d51e54-4a86-4b9f-82cc-cbd453110b19"} {"level":"info","ts":1789669023.2803717,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9ba8e55654aab812a6e71e04d31c3357","span_id":"200757c7cb97cd54","x-request-id":"01d51e54-4a86-4b9f-82cc-cbd453110b19","EoS":true} {"level":"info","ts":1789669023.280421,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9ba8e55654aab812a6e71e04d31c3357","span_id":"200757c7cb97cd54","x-request-id":"01d51e54-4a86-4b9f-82cc-cbd453110b19"} {"level":"info","ts":1789669023.4249537,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789669023.4250727,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"719bedc49a1f0c91f441ce5254c03aa8","span_id":"1e8fb00247cbad3a","x-request-id":"b5bac7b0-e915-469d-8448-256fbf27a193"} {"level":"info","ts":1789669023.425093,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"719bedc49a1f0c91f441ce5254c03aa8","span_id":"1e8fb00247cbad3a","x-request-id":"b5bac7b0-e915-469d-8448-256fbf27a193"} {"level":"info","ts":1789669023.4251099,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"719bedc49a1f0c91f441ce5254c03aa8","span_id":"1e8fb00247cbad3a","x-request-id":"b5bac7b0-e915-469d-8448-256fbf27a193","EoS":true} {"level":"info","ts":1789669023.4251575,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"719bedc49a1f0c91f441ce5254c03aa8","span_id":"1e8fb00247cbad3a","x-request-id":"b5bac7b0-e915-469d-8448-256fbf27a193"} {"level":"info","ts":1789669023.571933,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789669023.5720165,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fd753b126887ff8e6ac2eba688f25771","span_id":"cbf0b9ba5ba1cce7","x-request-id":"0e0f35d6-32e3-4697-8877-c84aaaffbcdd"} {"level":"info","ts":1789669023.5720308,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fd753b126887ff8e6ac2eba688f25771","span_id":"cbf0b9ba5ba1cce7","x-request-id":"0e0f35d6-32e3-4697-8877-c84aaaffbcdd"} {"level":"info","ts":1789669023.5720432,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fd753b126887ff8e6ac2eba688f25771","span_id":"cbf0b9ba5ba1cce7","x-request-id":"0e0f35d6-32e3-4697-8877-c84aaaffbcdd","EoS":true} {"level":"info","ts":1789669023.5720844,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fd753b126887ff8e6ac2eba688f25771","span_id":"cbf0b9ba5ba1cce7","x-request-id":"0e0f35d6-32e3-4697-8877-c84aaaffbcdd"} {"level":"error","ts":1789669024.2079153,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789669039.210006,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670142.3386168,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670142.3387399,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.3387554,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.3387682,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","EoS":true} {"level":"info","ts":1789670142.3387945,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","header":"x-maas-username"} {"level":"info","ts":1789670142.338805,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","header":"x-maas-group"} {"level":"info","ts":1789670142.338814,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","header":"x-maas-keyname"} {"level":"info","ts":1789670142.3388238,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","header":"x-maas-subscription"} {"level":"info","ts":1789670142.3388424,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789670142.338923,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.3405228,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.340545,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.3405669,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","EoS":false} {"level":"info","ts":1789670142.3405845,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.34065,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8","EoS":true} {"level":"info","ts":1789670142.3406632,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ad1b1a750ef29d8fdf121483a29f2f78","span_id":"def51172eaa0ae83","x-request-id":"7698f481-d6b2-42ec-bc09-301016e3e8a8"} {"level":"info","ts":1789670142.3787646,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670142.378854,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"info","ts":1789670142.3788803,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"info","ts":1789670142.3788931,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","EoS":true} {"level":"info","ts":1789670142.378922,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","header":"x-maas-group"} {"level":"info","ts":1789670142.3789344,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","header":"x-maas-keyname"} {"level":"info","ts":1789670142.3789475,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","header":"x-maas-subscription"} {"level":"info","ts":1789670142.3789525,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","header":"x-maas-username"} {"level":"info","ts":1789670142.378966,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789670142.3789937,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"info","ts":1789670142.379804,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"info","ts":1789670142.3798287,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"info","ts":1789670142.3798485,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","EoS":false} {"level":"info","ts":1789670142.3798807,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"info","ts":1789670142.3799102,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012","EoS":true} {"level":"info","ts":1789670142.3799212,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1084f51574be284079ba4b3c329acb08","span_id":"9fde96dbaf951cc5","x-request-id":"ed94da8b-3c49-4af9-84d0-5ebd62872012"} {"level":"error","ts":1789670154.3368514,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670468.1162996,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"405c4243-c07e-4599-b8f0-f4d600de7f7b"} {"level":"info","ts":1789670468.11632,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3737c79e-995f-49f4-8790-af913eb71077","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789670468.1163766,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3737c79e-995f-49f4-8790-af913eb71077","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.1164014,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3737c79e-995f-49f4-8790-af913eb71077"} {"level":"info","ts":1789670468.1240244,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3c35cebe-3d01-4140-8af2-166f1a3ee4c8","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789670468.124057,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3c35cebe-3d01-4140-8af2-166f1a3ee4c8","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.1240745,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3c35cebe-3d01-4140-8af2-166f1a3ee4c8"} {"level":"error","ts":1789670468.1241114,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"405c4243-c07e-4599-b8f0-f4d600de7f7b","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.132,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"405c4243-c07e-4599-b8f0-f4d600de7f7b","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.1320558,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4360598e-d5d8-45ed-b27c-f4bafe676b82"} {"level":"info","ts":1789670468.1320064,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0c3a19e3-00d1-4f57-b1b2-e7b76a0c18d7","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789670468.1321087,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0c3a19e3-00d1-4f57-b1b2-e7b76a0c18d7","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.132129,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0c3a19e3-00d1-4f57-b1b2-e7b76a0c18d7"} {"level":"error","ts":1789670468.1320744,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4360598e-d5d8-45ed-b27c-f4bafe676b82","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.137919,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4360598e-d5d8-45ed-b27c-f4bafe676b82","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.148134,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4fc2bcc6-c18c-451a-85cb-6413049453bb"} {"level":"error","ts":1789670468.1481624,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4fc2bcc6-c18c-451a-85cb-6413049453bb","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.1538448,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4fc2bcc6-c18c-451a-85cb-6413049453bb","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.1739879,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f7cced19-5d9f-4ee2-8c54-af10cb52ebd2"} {"level":"error","ts":1789670468.1740174,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f7cced19-5d9f-4ee2-8c54-af10cb52ebd2","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.1794105,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f7cced19-5d9f-4ee2-8c54-af10cb52ebd2","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.2205164,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d4718bd3-a102-4484-9dd0-b130716a4a2c"} {"level":"error","ts":1789670468.2205534,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d4718bd3-a102-4484-9dd0-b130716a4a2c","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.2252634,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d4718bd3-a102-4484-9dd0-b130716a4a2c","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.2689018,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"594f2d09-9bae-4b2e-b410-8a93bc3be4db"} {"level":"info","ts":1789670468.2689352,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9916ede3-1f0e-422f-adeb-720529a6e2df","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789670468.2689383,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"594f2d09-9bae-4b2e-b410-8a93bc3be4db","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.268964,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9916ede3-1f0e-422f-adeb-720529a6e2df","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.2689857,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9916ede3-1f0e-422f-adeb-720529a6e2df"} {"level":"error","ts":1789670468.2734556,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"594f2d09-9bae-4b2e-b410-8a93bc3be4db","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.4336293,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0197810d-c191-4c4f-a1f9-efe4eab0f3da"} {"level":"error","ts":1789670468.433685,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0197810d-c191-4c4f-a1f9-efe4eab0f3da","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.4391048,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0197810d-c191-4c4f-a1f9-efe4eab0f3da","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670468.759624,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"14985f84-5772-426c-9739-84f3d6e7a50a"} {"level":"error","ts":1789670468.7596712,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"14985f84-5772-426c-9739-84f3d6e7a50a","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670468.7650225,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"14985f84-5772-426c-9739-84f3d6e7a50a","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670469.4060557,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2389a5c3-bd48-4ed0-9515-e0d9c7046610"} {"level":"error","ts":1789670469.406111,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2389a5c3-bd48-4ed0-9515-e0d9c7046610","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670469.4117832,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2389a5c3-bd48-4ed0-9515-e0d9c7046610","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670470.6919854,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6a647ca6-ccfb-453c-83c1-fa89af703a69"} {"level":"error","ts":1789670470.6920393,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6a647ca6-ccfb-453c-83c1-fa89af703a69","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670470.6975641,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6a647ca6-ccfb-453c-83c1-fa89af703a69","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670473.2581985,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c4c95298-69eb-486c-8af1-70fb82411664"} {"level":"error","ts":1789670473.258257,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c4c95298-69eb-486c-8af1-70fb82411664","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789670473.2635152,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c4c95298-69eb-486c-8af1-70fb82411664","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670473.2695403,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e4a7fb38-e5cd-46cc-9558-40ff705cdf01","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789670473.2695909,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e4a7fb38-e5cd-46cc-9558-40ff705cdf01","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789670473.2696097,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e4a7fb38-e5cd-46cc-9558-40ff705cdf01"} {"level":"info","ts":1789670473.5635295,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670473.5636353,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.563648,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.5636606,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","EoS":true} {"level":"info","ts":1789670473.5636835,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","header":"x-maas-group"} {"level":"info","ts":1789670473.5636945,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","header":"x-maas-username"} {"level":"info","ts":1789670473.5636992,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","header":"x-maas-subscription"} {"level":"info","ts":1789670473.5637038,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","header":"x-maas-keyname"} {"level":"info","ts":1789670473.5637197,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670473.5637445,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.5647216,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.5647423,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.5647643,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","EoS":false} {"level":"info","ts":1789670473.5647795,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.5648143,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c","EoS":true} {"level":"info","ts":1789670473.564825,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a7a2cab2cd790de805db179c57661454","span_id":"b62b1c4901ed0461","x-request-id":"e4d4d224-f247-4012-bfb0-592778fa2d1c"} {"level":"info","ts":1789670473.683719,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670473.683819,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.6838317,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.683845,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","EoS":true} {"level":"info","ts":1789670473.6838934,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","header":"x-maas-username"} {"level":"info","ts":1789670473.6839054,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","header":"x-maas-group"} {"level":"info","ts":1789670473.6839106,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","header":"x-maas-subscription"} {"level":"info","ts":1789670473.6839154,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","header":"x-maas-keyname"} {"level":"info","ts":1789670473.68393,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670473.6839578,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.6847835,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.6848018,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.6848164,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","EoS":false} {"level":"info","ts":1789670473.6848264,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.6848564,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71","EoS":true} {"level":"info","ts":1789670473.6848807,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b98e6404252b044d4e22928eb0297675","span_id":"120fadf1d93b9257","x-request-id":"ebcc00c9-7498-41d3-95e1-676e90afef71"} {"level":"info","ts":1789670473.7703185,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670473.7704265,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.770444,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.7704582,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","EoS":true} {"level":"info","ts":1789670473.770482,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","header":"x-maas-subscription"} {"level":"info","ts":1789670473.7704926,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","header":"x-maas-group"} {"level":"info","ts":1789670473.770499,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","header":"x-maas-keyname"} {"level":"info","ts":1789670473.7705038,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","header":"x-maas-username"} {"level":"info","ts":1789670473.7705183,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670473.7705445,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.7715695,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.7715871,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.7716074,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","EoS":false} {"level":"info","ts":1789670473.7716236,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.7716596,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc","EoS":true} {"level":"info","ts":1789670473.7716699,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2c3b32c9c96cdb130237adb8f19f81fd","span_id":"f7b7d92f6c8cc48c","x-request-id":"1080ac32-a647-4cd5-843f-87d00c4417dc"} {"level":"info","ts":1789670473.908409,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8c796807-b2f0-4f9c-8414-3cb6ea6c8a0e"} {"level":"info","ts":1789670473.9084477,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1aa6ae85-e6be-4c7e-8401-661ae0516ed9","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789670473.9084728,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1aa6ae85-e6be-4c7e-8401-661ae0516ed9","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789670484.3748045,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670496.718067,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670496.7181792,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.7181966,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.7182117,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","EoS":true} {"level":"info","ts":1789670496.718237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","header":"x-maas-subscription"} {"level":"info","ts":1789670496.7182474,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","header":"x-maas-username"} {"level":"info","ts":1789670496.718253,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","header":"x-maas-group"} {"level":"info","ts":1789670496.7182584,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","header":"x-maas-keyname"} {"level":"info","ts":1789670496.7182748,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670496.7182996,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.7310238,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.7310462,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.7310617,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","EoS":false} {"level":"info","ts":1789670496.7310739,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.731095,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950","EoS":true} {"level":"info","ts":1789670496.7311046,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"47b543699a0a9b7f6f4403accac550b3","span_id":"f0e482d735fc8a31","x-request-id":"8690baef-68e6-4bd8-92bf-3262bb73b950"} {"level":"info","ts":1789670496.8705177,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670496.8706167,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670496.8706295,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670496.8706417,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","EoS":true} {"level":"info","ts":1789670496.8706691,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","header":"x-maas-group"} {"level":"info","ts":1789670496.8706813,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","header":"x-maas-keyname"} {"level":"info","ts":1789670496.8706875,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","header":"x-maas-username"} {"level":"info","ts":1789670496.870693,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","header":"x-maas-subscription"} {"level":"info","ts":1789670496.8707087,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670496.8707318,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670496.871594,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670496.8716142,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670496.8716295,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","EoS":false} {"level":"info","ts":1789670496.871644,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670496.8716729,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7","EoS":true} {"level":"info","ts":1789670496.8716857,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2c67add5505b7a7b7c701adf0b32dc3f","span_id":"db07c7c210daecce","x-request-id":"9378609d-7ca9-4705-b741-482e259c38b7"} {"level":"info","ts":1789670497.012821,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670497.0129395,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.0129547,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.0129724,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","EoS":true} {"level":"info","ts":1789670497.0130045,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","header":"x-maas-keyname"} {"level":"info","ts":1789670497.0130157,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","header":"x-maas-username"} {"level":"info","ts":1789670497.0130234,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","header":"x-maas-group"} {"level":"info","ts":1789670497.013031,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","header":"x-maas-subscription"} {"level":"info","ts":1789670497.0130532,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670497.0130858,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.0151865,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.0152144,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.0152295,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","EoS":false} {"level":"info","ts":1789670497.015241,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.0152779,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a","EoS":true} {"level":"info","ts":1789670497.015289,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f6729be2c9ed4b9a4ff4dec83bf79839","span_id":"93fc54bf55591516","x-request-id":"1e6f2ba7-086a-4e43-bfe2-c31b1391e65a"} {"level":"info","ts":1789670497.1501615,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670497.150265,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.1502829,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.1502967,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","EoS":true} {"level":"info","ts":1789670497.150317,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","header":"x-maas-username"} {"level":"info","ts":1789670497.1503284,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","header":"x-maas-keyname"} {"level":"info","ts":1789670497.1503336,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","header":"x-maas-subscription"} {"level":"info","ts":1789670497.1503396,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","header":"x-maas-group"} {"level":"info","ts":1789670497.1503541,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670497.1503785,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.1512332,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.1512532,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.1512737,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","EoS":false} {"level":"info","ts":1789670497.15129,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.151319,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a","EoS":true} {"level":"info","ts":1789670497.1513298,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"97adb741e62b173475e18e9d232a99ca","span_id":"a063c04020c3764c","x-request-id":"e5ea5b61-a07a-45dd-b7d1-15dee051808a"} {"level":"info","ts":1789670497.290381,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670497.2904887,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670497.2905033,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670497.2905223,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","EoS":true} {"level":"info","ts":1789670497.2905548,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","header":"x-maas-group"} {"level":"info","ts":1789670497.2905679,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","header":"x-maas-keyname"} {"level":"info","ts":1789670497.2905774,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","header":"x-maas-username"} {"level":"info","ts":1789670497.2905853,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","header":"x-maas-subscription"} {"level":"info","ts":1789670497.2906067,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670497.2906415,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670497.2915282,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670497.291548,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670497.2915614,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","EoS":false} {"level":"info","ts":1789670497.2915738,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670497.2916024,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575","EoS":true} {"level":"info","ts":1789670497.291613,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a71a50c49b9a843c6eac7a9d0efd72a0","span_id":"e072b28d7a5b6db5","x-request-id":"ad9f12f0-a86d-4a7b-9f55-4b2201a8c575"} {"level":"info","ts":1789670506.117496,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670506.1175988,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.117614,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.1176329,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","EoS":true} {"level":"info","ts":1789670506.117667,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","header":"x-maas-subscription"} {"level":"info","ts":1789670506.1176789,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","header":"x-maas-keyname"} {"level":"info","ts":1789670506.1176872,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","header":"x-maas-username"} {"level":"info","ts":1789670506.1176968,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","header":"x-maas-group"} {"level":"info","ts":1789670506.1177185,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670506.117749,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.1186225,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.1186433,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.1186588,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","EoS":false} {"level":"info","ts":1789670506.1186757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.118702,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229","EoS":true} {"level":"info","ts":1789670506.1187115,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"282f856d16f6058ef534f4d6a2afd15a","span_id":"146fad896a4c99c8","x-request-id":"8c71c7c7-4e6e-4418-ab22-33dcca051229"} {"level":"info","ts":1789670506.152664,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670506.152768,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.1527817,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.1528022,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","EoS":true} {"level":"info","ts":1789670506.1528358,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","header":"x-maas-keyname"} {"level":"info","ts":1789670506.1528478,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","header":"x-maas-subscription"} {"level":"info","ts":1789670506.152857,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","header":"x-maas-username"} {"level":"info","ts":1789670506.1528826,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","header":"x-maas-group"} {"level":"info","ts":1789670506.1529033,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670506.1529343,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.1536942,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.1537113,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.1537225,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","EoS":false} {"level":"info","ts":1789670506.153734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.1537538,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3","EoS":true} {"level":"info","ts":1789670506.1537628,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f938fab777eab6ba3eae3f5f629ec1a9","span_id":"85aa750bb678c06d","x-request-id":"d638a7c0-05af-4a2e-bfaf-f00c92f816d3"} {"level":"info","ts":1789670506.2507343,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670506.2508328,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393"} {"level":"info","ts":1789670506.251599,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393"} {"level":"info","ts":1789670506.2516208,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393"} {"level":"info","ts":1789670506.2516332,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393","EoS":false} {"level":"info","ts":1789670506.2516448,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393"} {"level":"info","ts":1789670506.2516694,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393","EoS":true} {"level":"info","ts":1789670506.2516787,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"08398506f314effb453fafe45f3f5f5b","span_id":"4b50c6cd8eafb53c","x-request-id":"f17e9ab9-f44f-42a5-b20b-908ed8aa5393"} {"level":"info","ts":1789670507.0209746,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670507.0210645,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.0210795,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.021092,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","EoS":true} {"level":"info","ts":1789670507.0211134,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","header":"x-maas-username"} {"level":"info","ts":1789670507.0211234,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","header":"x-maas-group"} {"level":"info","ts":1789670507.0211291,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","header":"x-maas-subscription"} {"level":"info","ts":1789670507.0211337,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","header":"x-maas-keyname"} {"level":"info","ts":1789670507.0211475,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670507.0211723,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.0220056,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.0220258,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.0220394,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","EoS":false} {"level":"info","ts":1789670507.0220504,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.022077,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7","EoS":true} {"level":"info","ts":1789670507.0220895,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"89721babea826a67b43c1fb41d8efdbe","span_id":"de16f520ef98a4b3","x-request-id":"06259ee1-c1c6-4308-8d49-e3eb8afcb9c7"} {"level":"info","ts":1789670507.9735014,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670507.9735992,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670507.9736133,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670507.9736276,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","EoS":true} {"level":"info","ts":1789670507.9736617,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","header":"x-maas-group"} {"level":"info","ts":1789670507.9736714,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","header":"x-maas-keyname"} {"level":"info","ts":1789670507.9736772,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","header":"x-maas-subscription"} {"level":"info","ts":1789670507.9736817,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","header":"x-maas-username"} {"level":"info","ts":1789670507.973696,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670507.9737315,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670507.9745696,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670507.9745889,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670507.974603,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","EoS":false} {"level":"info","ts":1789670507.974615,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670507.9746423,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05","EoS":true} {"level":"info","ts":1789670507.9746518,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"704b8eb2cc759a5fdf0f27ee2efb4fb7","span_id":"40cde1acb3f1ddd8","x-request-id":"c0438954-87a8-40e6-b84f-58da527f2d05"} {"level":"info","ts":1789670508.696421,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670508.6965063,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"info","ts":1789670508.6965175,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"info","ts":1789670508.696529,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","EoS":true} {"level":"info","ts":1789670508.696552,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","header":"x-maas-username"} {"level":"info","ts":1789670508.6965613,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","header":"x-maas-keyname"} {"level":"info","ts":1789670508.696566,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","header":"x-maas-subscription"} {"level":"info","ts":1789670508.696571,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","header":"x-maas-group"} {"level":"info","ts":1789670508.6965847,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670508.6966088,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"info","ts":1789670508.6974213,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"info","ts":1789670508.6974416,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"info","ts":1789670508.6974556,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","EoS":false} {"level":"info","ts":1789670508.6974704,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"info","ts":1789670508.6974957,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf","EoS":true} {"level":"info","ts":1789670508.6975043,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5f75191accc0b169ee350286dbf98e40","span_id":"99cb70bd6b4f4cce","x-request-id":"f2cbd375-cd53-413d-ad48-0c3df51e3faf"} {"level":"error","ts":1789670509.3764477,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670509.76529,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670509.7653816,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"info","ts":1789670509.7653935,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"info","ts":1789670509.7654047,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","EoS":true} {"level":"info","ts":1789670509.765427,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","header":"x-maas-subscription"} {"level":"info","ts":1789670509.7654374,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","header":"x-maas-group"} {"level":"info","ts":1789670509.765443,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","header":"x-maas-keyname"} {"level":"info","ts":1789670509.765448,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","header":"x-maas-username"} {"level":"info","ts":1789670509.7654629,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670509.7654872,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"info","ts":1789670509.7663057,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"info","ts":1789670509.7663264,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"info","ts":1789670509.766341,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","EoS":false} {"level":"info","ts":1789670509.766355,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"info","ts":1789670509.7663796,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446","EoS":true} {"level":"info","ts":1789670509.7663896,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7115b060baa013ca379e33944e4cab27","span_id":"3309827af312581a","x-request-id":"c2ada054-5129-4b98-b869-03ad25373446"} {"level":"error","ts":1789670519.376854,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670527.4073818,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670527.4074945,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"info","ts":1789670527.4075084,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"info","ts":1789670527.4075243,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","EoS":true} {"level":"info","ts":1789670527.407552,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","header":"x-maas-subscription"} {"level":"info","ts":1789670527.4075623,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","header":"x-maas-username"} {"level":"info","ts":1789670527.407568,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","header":"x-maas-keyname"} {"level":"info","ts":1789670527.4075732,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","header":"x-maas-group"} {"level":"info","ts":1789670527.4075878,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670527.4076214,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"info","ts":1789670527.4085047,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"info","ts":1789670527.408532,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"info","ts":1789670527.408554,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","EoS":false} {"level":"info","ts":1789670527.4085686,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"info","ts":1789670527.4085956,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d","EoS":true} {"level":"info","ts":1789670527.4086049,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1cfbd7960eca9fec7165930ba975969c","span_id":"dcf49bf0faf68bc7","x-request-id":"4e3b4f97-7249-4f57-a7ed-787483d4bd5d"} {"level":"error","ts":1789670534.377957,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789670549.3789248,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670616.5542102,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670616.5543447,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"info","ts":1789670616.5543609,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"info","ts":1789670616.5543778,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","EoS":true} {"level":"info","ts":1789670616.5544112,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","header":"x-maas-group"} {"level":"info","ts":1789670616.5544252,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","header":"x-maas-username"} {"level":"info","ts":1789670616.5544338,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","header":"x-maas-subscription"} {"level":"info","ts":1789670616.5544407,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","header":"x-maas-keyname"} {"level":"info","ts":1789670616.5544612,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789670616.5544958,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"info","ts":1789670616.5677395,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"info","ts":1789670616.5677643,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"info","ts":1789670616.5677795,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","EoS":false} {"level":"info","ts":1789670616.5677931,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"info","ts":1789670616.5678165,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a","EoS":true} {"level":"info","ts":1789670616.5678256,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7b4404fdd714a9ed9623cae28c93155e","span_id":"33f03e78765abe8c","x-request-id":"ff39eb4d-e275-4ec7-b069-55a4150ec39a"} {"level":"error","ts":1789670629.388303,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670709.177713,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670709.1778054,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.1778193,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.1778305,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","EoS":true} {"level":"info","ts":1789670709.1778526,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","header":"x-maas-keyname"} {"level":"info","ts":1789670709.1778784,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","header":"x-maas-group"} {"level":"info","ts":1789670709.1778846,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","header":"x-maas-username"} {"level":"info","ts":1789670709.17789,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","header":"x-maas-subscription"} {"level":"info","ts":1789670709.1779037,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670709.177928,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.1787376,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.1787717,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.1787918,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","EoS":false} {"level":"info","ts":1789670709.178811,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.1788392,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d","EoS":true} {"level":"info","ts":1789670709.1788495,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"245b65333728b0fdbc7d662d81b5ba1a","span_id":"45cbcf61ce74c1eb","x-request-id":"3d1134eb-8de1-408e-9f4f-abc14df3f45d"} {"level":"info","ts":1789670709.2110095,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670709.2111037,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"info","ts":1789670709.2111177,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"info","ts":1789670709.2111306,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","EoS":true} {"level":"info","ts":1789670709.2111516,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","header":"x-maas-username"} {"level":"info","ts":1789670709.2111619,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","header":"x-maas-group"} {"level":"info","ts":1789670709.2111678,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","header":"x-maas-subscription"} {"level":"info","ts":1789670709.2111726,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","header":"x-maas-keyname"} {"level":"info","ts":1789670709.2111864,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789670709.2112098,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"info","ts":1789670709.2120192,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"info","ts":1789670709.2120411,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"info","ts":1789670709.2120562,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","EoS":false} {"level":"info","ts":1789670709.2120674,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"info","ts":1789670709.2121606,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170","EoS":true} {"level":"info","ts":1789670709.212174,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"00636c1ea82e8f723ae289f2f8d044d8","span_id":"6cb330258c053eea","x-request-id":"46b82698-19c9-4192-85c3-498c1d3a2170"} {"level":"error","ts":1789670719.3965957,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670723.0449636,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670723.0450873,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.0451071,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.0451262,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","EoS":true} {"level":"info","ts":1789670723.0451543,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","header":"x-maas-keyname"} {"level":"info","ts":1789670723.0451665,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","header":"x-maas-group"} {"level":"info","ts":1789670723.0451753,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","header":"x-maas-username"} {"level":"info","ts":1789670723.045183,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","header":"x-maas-subscription"} {"level":"info","ts":1789670723.0452013,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789670723.045238,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.057551,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.057577,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.0575995,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","EoS":false} {"level":"info","ts":1789670723.0576186,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.0576558,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","EoS":false} {"level":"info","ts":1789670723.0576718,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.0576901,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d","EoS":true} {"level":"info","ts":1789670723.057703,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d829cc0a40dd1de36fa50f5226062b8a","span_id":"394c056bf3b52692","x-request-id":"95fa7629-80be-4a26-b352-1bcd595ebd1d"} {"level":"info","ts":1789670723.1956959,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670723.1957917,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.195803,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.1958144,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","EoS":true} {"level":"info","ts":1789670723.195835,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","header":"x-maas-keyname"} {"level":"info","ts":1789670723.1958444,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","header":"x-maas-group"} {"level":"info","ts":1789670723.1958492,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","header":"x-maas-username"} {"level":"info","ts":1789670723.1958535,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","header":"x-maas-subscription"} {"level":"info","ts":1789670723.195882,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789670723.1959102,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.19773,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.1977496,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.1977677,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","EoS":false} {"level":"info","ts":1789670723.1977823,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.1978033,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","EoS":false} {"level":"info","ts":1789670723.197814,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.1978242,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec","EoS":true} {"level":"info","ts":1789670723.1978312,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d9f3ef5bcb71e7b7a005bd3afcba1eb","span_id":"1e847743a1795028","x-request-id":"0dcdad6c-0b91-4dc5-b444-3c80d5f2c0ec"} {"level":"info","ts":1789670723.337088,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670723.3371766,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"info","ts":1789670723.3371902,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"info","ts":1789670723.3372016,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","EoS":true} {"level":"info","ts":1789670723.3372247,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","header":"x-maas-username"} {"level":"info","ts":1789670723.337235,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","header":"x-maas-group"} {"level":"info","ts":1789670723.3372412,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","header":"x-maas-subscription"} {"level":"info","ts":1789670723.3372457,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","header":"x-maas-keyname"} {"level":"info","ts":1789670723.3372598,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789670723.3372874,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"info","ts":1789670723.3380675,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"info","ts":1789670723.338087,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"info","ts":1789670723.3381097,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","EoS":false} {"level":"info","ts":1789670723.338127,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"info","ts":1789670723.338163,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9","EoS":true} {"level":"info","ts":1789670723.3381734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82608d61eba6d216a9d14d5191247b7a","span_id":"fb61ef33a05e5351","x-request-id":"8334125b-8f39-460e-86fe-c3f8684d3bf9"} {"level":"error","ts":1789670734.3979242,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789670736.178577,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789670736.1787097,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"info","ts":1789670736.1787288,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"info","ts":1789670736.1787498,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","EoS":true} {"level":"info","ts":1789670736.1787796,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","header":"x-maas-username"} {"level":"info","ts":1789670736.178795,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","header":"x-maas-group"} {"level":"info","ts":1789670736.178803,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","header":"x-maas-subscription"} {"level":"info","ts":1789670736.1788106,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","header":"x-maas-keyname"} {"level":"info","ts":1789670736.1788306,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789670736.1788833,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"info","ts":1789670736.1805363,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"info","ts":1789670736.1805642,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"info","ts":1789670736.1805966,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","EoS":false} {"level":"info","ts":1789670736.1806202,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"info","ts":1789670736.1806574,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f","EoS":true} {"level":"info","ts":1789670736.1806684,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"273e77859a680607283de49bb9b66e4e","span_id":"b7f05b6cc34447c0","x-request-id":"f89527ca-84c1-47d9-992d-79d26a5e129f"} {"level":"error","ts":1789670749.3992794,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}