{"level":"info","ts":1789682180.3478239,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"967b0c141834cb8988c799406d448fc3","span_id":"308df263ccbc8814","x-request-id":"9233acb4-b83d-9c42-86a6-b00bc4e63514","EoS":true} {"level":"info","ts":1789682180.3478315,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"967b0c141834cb8988c799406d448fc3","span_id":"308df263ccbc8814","x-request-id":"9233acb4-b83d-9c42-86a6-b00bc4e63514"} {"level":"info","ts":1789682180.3840182,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789682180.3841145,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.384128,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.3841393,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","EoS":true} {"level":"info","ts":1789682180.3841639,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","header":"x-maas-group"} {"level":"info","ts":1789682180.384174,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","header":"x-maas-username"} {"level":"info","ts":1789682180.3841798,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","header":"x-maas-subscription"} {"level":"info","ts":1789682180.3841846,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","header":"x-maas-keyname"} {"level":"info","ts":1789682180.3842015,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789682180.3842278,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.3851528,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.385164,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.3851748,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","EoS":false} {"level":"info","ts":1789682180.385185,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.3852048,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1","EoS":true} {"level":"info","ts":1789682180.385214,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ffa2a313d70daae952653d1407c724a2","span_id":"53875cd5496b8e2d","x-request-id":"12a70afd-c57d-453e-86f4-9d613d2ee5d1"} {"level":"info","ts":1789682180.436098,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789682180.4361966,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.4362102,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.4362226,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","EoS":true} {"level":"info","ts":1789682180.4362452,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","header":"x-maas-keyname"} {"level":"info","ts":1789682180.4362557,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","header":"x-maas-username"} {"level":"info","ts":1789682180.4362605,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","header":"x-maas-group"} {"level":"info","ts":1789682180.4362648,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","header":"x-maas-subscription"} {"level":"info","ts":1789682180.43628,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789682180.4363391,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.4373548,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.437376,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.4373896,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","EoS":false} {"level":"info","ts":1789682180.437402,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.4374275,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18","EoS":true} {"level":"info","ts":1789682180.4374368,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"54875817dd27aa571cdb9cc19c20cef0","span_id":"b30e0ea7c2a783b0","x-request-id":"c099693f-7ec7-44de-a2fa-29c263489e18"} {"level":"info","ts":1789682180.4746823,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789682180.4747753,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"info","ts":1789682180.474788,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"info","ts":1789682180.4747992,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","EoS":true} {"level":"info","ts":1789682180.4748237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","header":"x-maas-group"} {"level":"info","ts":1789682180.4748333,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","header":"x-maas-username"} {"level":"info","ts":1789682180.474838,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","header":"x-maas-subscription"} {"level":"info","ts":1789682180.4748442,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","header":"x-maas-keyname"} {"level":"info","ts":1789682180.4748569,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789682180.4748824,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"info","ts":1789682180.4758477,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"info","ts":1789682180.4758668,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"info","ts":1789682180.475881,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","EoS":false} {"level":"info","ts":1789682180.4758925,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"info","ts":1789682180.4759173,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651","EoS":true} {"level":"info","ts":1789682180.4759269,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"eb8594820756ea7a009843e981f270c6","span_id":"3fee6e232418a25b","x-request-id":"a0a4627f-f000-4271-b0a7-1d1acfda2651"} {"level":"error","ts":1789682194.9462178,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789682197.7013135,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789682197.701411,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c2e7a9c0b21281ef1571ce461df29b55","span_id":"7451f181895b8e8a","x-request-id":"89ef3d3c-5525-48de-b4dd-16837e07639c"} {"level":"info","ts":1789682197.7014241,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c2e7a9c0b21281ef1571ce461df29b55","span_id":"7451f181895b8e8a","x-request-id":"89ef3d3c-5525-48de-b4dd-16837e07639c"} {"level":"info","ts":1789682197.701436,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c2e7a9c0b21281ef1571ce461df29b55","span_id":"7451f181895b8e8a","x-request-id":"89ef3d3c-5525-48de-b4dd-16837e07639c","EoS":true} {"level":"info","ts":1789682197.7014878,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c2e7a9c0b21281ef1571ce461df29b55","span_id":"7451f181895b8e8a","x-request-id":"89ef3d3c-5525-48de-b4dd-16837e07639c"} {"level":"info","ts":1789682197.856853,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789682197.8569694,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9b122d6a64de32ac175691164bbf93d1","span_id":"6c8f881d38bd4a68","x-request-id":"199864ac-9a3f-4723-a5bd-681140650a17"} {"level":"info","ts":1789682197.8569915,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9b122d6a64de32ac175691164bbf93d1","span_id":"6c8f881d38bd4a68","x-request-id":"199864ac-9a3f-4723-a5bd-681140650a17"} {"level":"info","ts":1789682197.8570173,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9b122d6a64de32ac175691164bbf93d1","span_id":"6c8f881d38bd4a68","x-request-id":"199864ac-9a3f-4723-a5bd-681140650a17","EoS":true} {"level":"info","ts":1789682197.8570743,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9b122d6a64de32ac175691164bbf93d1","span_id":"6c8f881d38bd4a68","x-request-id":"199864ac-9a3f-4723-a5bd-681140650a17"} {"level":"info","ts":1789682198.0197694,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789682198.0198922,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"867d7dfd653b0d8f2e25b8432e27b902","span_id":"ed71b6c69dd41763","x-request-id":"e6fff229-859f-440f-9837-6f603af63a05"} {"level":"info","ts":1789682198.0199087,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"867d7dfd653b0d8f2e25b8432e27b902","span_id":"ed71b6c69dd41763","x-request-id":"e6fff229-859f-440f-9837-6f603af63a05"} {"level":"info","ts":1789682198.0199273,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"867d7dfd653b0d8f2e25b8432e27b902","span_id":"ed71b6c69dd41763","x-request-id":"e6fff229-859f-440f-9837-6f603af63a05","EoS":true} {"level":"info","ts":1789682198.01998,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"867d7dfd653b0d8f2e25b8432e27b902","span_id":"ed71b6c69dd41763","x-request-id":"e6fff229-859f-440f-9837-6f603af63a05"} {"level":"error","ts":1789682209.9479392,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683384.1075063,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683384.1076183,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.1076305,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.1076424,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","EoS":true} {"level":"info","ts":1789683384.1076705,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","header":"x-maas-keyname"} {"level":"info","ts":1789683384.107681,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","header":"x-maas-subscription"} {"level":"info","ts":1789683384.1076868,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","header":"x-maas-group"} {"level":"info","ts":1789683384.1076908,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","header":"x-maas-username"} {"level":"info","ts":1789683384.1077056,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789683384.1077359,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.1086357,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.1086626,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.108678,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","EoS":false} {"level":"info","ts":1789683384.108689,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.1087112,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b","EoS":true} {"level":"info","ts":1789683384.1087203,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"92e63f4ab2ff0e45e37ec2404e9568bf","span_id":"3ce9b9dc66f1c17f","x-request-id":"97d8e642-3ebf-489e-870d-44106b56ca4b"} {"level":"info","ts":1789683384.147739,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683384.147844,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"info","ts":1789683384.1478555,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"info","ts":1789683384.1478677,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","EoS":true} {"level":"info","ts":1789683384.1478915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","header":"x-maas-group"} {"level":"info","ts":1789683384.1479025,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","header":"x-maas-subscription"} {"level":"info","ts":1789683384.1479073,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","header":"x-maas-username"} {"level":"info","ts":1789683384.1479118,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","header":"x-maas-keyname"} {"level":"info","ts":1789683384.147926,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789683384.147959,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"info","ts":1789683384.1488178,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"info","ts":1789683384.1488366,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"info","ts":1789683384.1488495,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","EoS":false} {"level":"info","ts":1789683384.148859,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"info","ts":1789683384.1488814,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20","EoS":true} {"level":"info","ts":1789683384.1488903,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5dc8f588f70960b4a0f366e43fd798be","span_id":"f8d97eef1180802a","x-request-id":"727467ad-6f3f-4fb9-8a45-dc320c781a20"} {"level":"error","ts":1789683395.0833516,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683718.3451078,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"57222310-2570-473b-b237-ffdfbc7531de"} {"level":"info","ts":1789683718.345167,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"addf357e-c160-44d8-b41b-6b37909fb62c","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789683718.345216,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"addf357e-c160-44d8-b41b-6b37909fb62c","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.3452353,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"addf357e-c160-44d8-b41b-6b37909fb62c"} {"level":"info","ts":1789683718.350866,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c3647911-1061-4edd-9cf7-fe27fe2fec44","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789683718.350897,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c3647911-1061-4edd-9cf7-fe27fe2fec44","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.3509173,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c3647911-1061-4edd-9cf7-fe27fe2fec44"} {"level":"error","ts":1789683718.3511727,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"57222310-2570-473b-b237-ffdfbc7531de","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.357038,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"eb6e474b-2c63-4744-bfd8-2dd04a891c83","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789683718.3570776,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"eb6e474b-2c63-4744-bfd8-2dd04a891c83","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.357101,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"eb6e474b-2c63-4744-bfd8-2dd04a891c83"} {"level":"error","ts":1789683718.3572,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"57222310-2570-473b-b237-ffdfbc7531de","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.3572752,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2484487e-fc80-4e88-a3cd-0d36a9636f02"} {"level":"error","ts":1789683718.3573272,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2484487e-fc80-4e88-a3cd-0d36a9636f02","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.362444,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2484487e-fc80-4e88-a3cd-0d36a9636f02","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.3726902,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"84c33847-c1a6-4aba-ac90-bce7e2084fb7"} {"level":"error","ts":1789683718.3727407,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"84c33847-c1a6-4aba-ac90-bce7e2084fb7","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.377978,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"84c33847-c1a6-4aba-ac90-bce7e2084fb7","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.3991303,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5766c0e7-9c92-459a-81de-f7b1f750386a"} {"level":"error","ts":1789683718.3991835,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5766c0e7-9c92-459a-81de-f7b1f750386a","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.4041388,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5766c0e7-9c92-459a-81de-f7b1f750386a","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.4453309,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"36b7d000-381d-42a9-832c-acdc418263a5"} {"level":"error","ts":1789683718.4453912,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"36b7d000-381d-42a9-832c-acdc418263a5","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.450278,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"36b7d000-381d-42a9-832c-acdc418263a5","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.4927,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2fc266e0-8eba-487c-8f7c-b61f7c1566c5","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789683718.492757,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2fc266e0-8eba-487c-8f7c-b61f7c1566c5","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.4927804,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2fc266e0-8eba-487c-8f7c-b61f7c1566c5"} {"level":"info","ts":1789683718.4927096,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"aef0fca1-9bac-43ba-89df-92a3bb5480c7"} {"level":"error","ts":1789683718.4928367,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"aef0fca1-9bac-43ba-89df-92a3bb5480c7","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.4980602,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"aef0fca1-9bac-43ba-89df-92a3bb5480c7","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.6583307,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3013f266-58dc-4522-8f96-8286d7869583"} {"level":"error","ts":1789683718.6584132,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3013f266-58dc-4522-8f96-8286d7869583","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.6645105,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3013f266-58dc-4522-8f96-8286d7869583","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683718.9850545,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ef180e98-28b7-47e9-8506-f5e8ebdbe7cc"} {"level":"error","ts":1789683718.9851167,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ef180e98-28b7-47e9-8506-f5e8ebdbe7cc","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683718.9905162,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ef180e98-28b7-47e9-8506-f5e8ebdbe7cc","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683719.631416,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"12ffc77c-7542-4539-ac85-e6845add43b4"} {"level":"error","ts":1789683719.631485,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"12ffc77c-7542-4539-ac85-e6845add43b4","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683719.6369565,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"12ffc77c-7542-4539-ac85-e6845add43b4","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683720.918095,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6cbd4efb-8894-44b8-abb3-fa7e34e1b0a2"} {"level":"error","ts":1789683720.9181504,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6cbd4efb-8894-44b8-abb3-fa7e34e1b0a2","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683720.9271574,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6cbd4efb-8894-44b8-abb3-fa7e34e1b0a2","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683723.487451,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ffd7dd08-ecb2-49fb-9813-ddb4cc288dc5"} {"level":"error","ts":1789683723.4875154,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ffd7dd08-ecb2-49fb-9813-ddb4cc288dc5","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683723.4936028,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4e205da2-d394-4eb6-9a77-b73a6d1038f4","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789683723.493648,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4e205da2-d394-4eb6-9a77-b73a6d1038f4","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683723.4936671,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4e205da2-d394-4eb6-9a77-b73a6d1038f4"} {"level":"error","ts":1789683723.4962716,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ffd7dd08-ecb2-49fb-9813-ddb4cc288dc5","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683728.4941678,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a718d8e6-021b-41c3-b5b3-7d84cf0b348d","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789683728.4942605,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a718d8e6-021b-41c3-b5b3-7d84cf0b348d","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683728.4943137,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a718d8e6-021b-41c3-b5b3-7d84cf0b348d"} {"level":"info","ts":1789683728.6174228,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d07fbf95-537b-406b-9e72-5296f1ba0ff1"} {"level":"error","ts":1789683728.617493,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d07fbf95-537b-406b-9e72-5296f1ba0ff1","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789683728.6227968,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d07fbf95-537b-406b-9e72-5296f1ba0ff1","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789683728.9496217,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683728.949744,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683728.9497588,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683728.9497778,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","EoS":true} {"level":"info","ts":1789683728.949816,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","header":"x-maas-keyname"} {"level":"info","ts":1789683728.949829,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","header":"x-maas-group"} {"level":"info","ts":1789683728.9498384,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","header":"x-maas-subscription"} {"level":"info","ts":1789683728.949847,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","header":"x-maas-username"} {"level":"info","ts":1789683728.9498703,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683728.949906,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683728.9507911,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683728.950811,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683728.9508252,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","EoS":false} {"level":"info","ts":1789683728.9508364,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683728.950864,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d","EoS":true} {"level":"info","ts":1789683728.9508765,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e2dbbb8a324335f556a48547bd93f4c0","span_id":"97fed60cc55d5bce","x-request-id":"7ecc4e3d-422f-48c1-b72c-bd264843748d"} {"level":"info","ts":1789683729.0543673,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683729.0545144,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.0545335,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.054552,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","EoS":true} {"level":"info","ts":1789683729.0545952,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","header":"x-maas-username"} {"level":"info","ts":1789683729.054618,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","header":"x-maas-keyname"} {"level":"info","ts":1789683729.0546262,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","header":"x-maas-group"} {"level":"info","ts":1789683729.0546336,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","header":"x-maas-subscription"} {"level":"info","ts":1789683729.0546696,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683729.054714,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.055616,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.0556366,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.055655,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","EoS":false} {"level":"info","ts":1789683729.0556738,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.0557106,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c","EoS":true} {"level":"info","ts":1789683729.0557225,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bba53b2841601a79b49eace6b5a4bd33","span_id":"31a98c1c9cbb9ffb","x-request-id":"a4f2c673-71be-45a1-afc7-e2184dbf0c9c"} {"level":"info","ts":1789683729.1513338,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683729.1514676,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.1514864,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.1515057,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","EoS":true} {"level":"info","ts":1789683729.1515398,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","header":"x-maas-group"} {"level":"info","ts":1789683729.1515543,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","header":"x-maas-keyname"} {"level":"info","ts":1789683729.1515625,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","header":"x-maas-username"} {"level":"info","ts":1789683729.1515696,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","header":"x-maas-subscription"} {"level":"info","ts":1789683729.1515894,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683729.1516244,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.152551,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.1525755,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.1525958,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","EoS":false} {"level":"info","ts":1789683729.152616,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.1526566,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11","EoS":true} {"level":"info","ts":1789683729.1526718,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e30ff2448a8f1f2a6cbfd30250f4ccf1","span_id":"ed952d4795768228","x-request-id":"9d865e24-edfa-47a1-99ab-51985afbbb11"} {"level":"info","ts":1789683729.337388,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5e60329f-ffc7-46cf-b642-b271e75be677","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789683729.337443,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5e60329f-ffc7-46cf-b642-b271e75be677","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789683729.3373883,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"503f0235-ff77-4af4-9e2a-cfc54a7f3946"} {"level":"error","ts":1789683740.1247265,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683747.106057,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683747.1061811,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.1061969,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.106216,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","EoS":true} {"level":"info","ts":1789683747.1062522,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","header":"x-maas-keyname"} {"level":"info","ts":1789683747.1062667,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","header":"x-maas-subscription"} {"level":"info","ts":1789683747.1062772,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","header":"x-maas-username"} {"level":"info","ts":1789683747.1063159,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","header":"x-maas-group"} {"level":"info","ts":1789683747.10634,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683747.10638,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.120216,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.120238,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.1202543,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","EoS":false} {"level":"info","ts":1789683747.120266,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.120311,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801","EoS":true} {"level":"info","ts":1789683747.1203237,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"672c7b8d9b3fda9551deb68e9b35210b","span_id":"e87ea7e540343fd6","x-request-id":"a134576a-291e-4f56-b481-5c07f26f8801"} {"level":"info","ts":1789683747.2580447,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683747.2581568,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.2581701,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.2581847,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","EoS":true} {"level":"info","ts":1789683747.2582202,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","header":"x-maas-keyname"} {"level":"info","ts":1789683747.2582304,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","header":"x-maas-group"} {"level":"info","ts":1789683747.2582355,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","header":"x-maas-subscription"} {"level":"info","ts":1789683747.2582402,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","header":"x-maas-username"} {"level":"info","ts":1789683747.2582545,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683747.258307,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.2593997,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.2594142,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.259427,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","EoS":false} {"level":"info","ts":1789683747.2594411,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.2594616,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c","EoS":true} {"level":"info","ts":1789683747.259471,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cfd1ea477c322372241cf7ce051f010c","span_id":"802a60d1e9183777","x-request-id":"71d7bc3e-08d8-46d7-84ad-9f5338fabf9c"} {"level":"info","ts":1789683747.4066467,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683747.406764,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.4067802,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.4068048,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","EoS":true} {"level":"info","ts":1789683747.4068875,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","header":"x-maas-subscription"} {"level":"info","ts":1789683747.4069023,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","header":"x-maas-username"} {"level":"info","ts":1789683747.4069111,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","header":"x-maas-keyname"} {"level":"info","ts":1789683747.4069192,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","header":"x-maas-group"} {"level":"info","ts":1789683747.406942,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683747.4069815,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.4080665,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.4080887,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.4080994,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","EoS":false} {"level":"info","ts":1789683747.4081132,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.4081373,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3","EoS":true} {"level":"info","ts":1789683747.408145,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a239a9a970c1b43aa9448d9eb6fddd96","span_id":"f0763e0e0fef3af5","x-request-id":"30706536-463f-4b70-b16d-60dcf187fcd3"} {"level":"info","ts":1789683747.5430722,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683747.5431736,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.5431852,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.5431979,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","EoS":true} {"level":"info","ts":1789683747.5432236,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","header":"x-maas-group"} {"level":"info","ts":1789683747.543233,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","header":"x-maas-username"} {"level":"info","ts":1789683747.5432396,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","header":"x-maas-subscription"} {"level":"info","ts":1789683747.5432446,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","header":"x-maas-keyname"} {"level":"info","ts":1789683747.5432599,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683747.543345,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.5456364,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.5456555,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.5456653,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","EoS":false} {"level":"info","ts":1789683747.5456774,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.545703,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7","EoS":true} {"level":"info","ts":1789683747.5457141,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8009868866b0ef36da8c3e3c51006070","span_id":"47e26e9cad92e486","x-request-id":"458bbd85-104b-4ca1-8a8d-8b5c20e645f7"} {"level":"info","ts":1789683747.6847088,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683747.684807,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"info","ts":1789683747.6848207,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"info","ts":1789683747.6848319,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","EoS":true} {"level":"info","ts":1789683747.6848574,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","header":"x-maas-keyname"} {"level":"info","ts":1789683747.6848671,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","header":"x-maas-username"} {"level":"info","ts":1789683747.6848717,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","header":"x-maas-subscription"} {"level":"info","ts":1789683747.684876,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","header":"x-maas-group"} {"level":"info","ts":1789683747.6849346,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683747.6849651,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"info","ts":1789683747.685969,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"info","ts":1789683747.6859882,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"info","ts":1789683747.6860018,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","EoS":false} {"level":"info","ts":1789683747.686013,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"info","ts":1789683747.6860378,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50","EoS":true} {"level":"info","ts":1789683747.6860487,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a96bef485777b77bd3874448a027b2bd","span_id":"09e4c574aec302c4","x-request-id":"024a4a7f-fd7e-4c9f-9910-4dcae8bcfc50"} {"level":"error","ts":1789683760.1264997,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683761.8144603,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683761.814666,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.8146894,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.8147097,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","EoS":true} {"level":"info","ts":1789683761.8147442,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","header":"x-maas-username"} {"level":"info","ts":1789683761.8147573,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","header":"x-maas-subscription"} {"level":"info","ts":1789683761.814765,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","header":"x-maas-keyname"} {"level":"info","ts":1789683761.8147728,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","header":"x-maas-group"} {"level":"info","ts":1789683761.8147957,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683761.8148327,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.8165808,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.8166075,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.8166268,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","EoS":false} {"level":"info","ts":1789683761.8166418,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.8166795,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec","EoS":true} {"level":"info","ts":1789683761.8166938,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b24b2cae0451ea1c447295e2d1c48b20","span_id":"10df56105649a376","x-request-id":"88afdbe3-2b50-4e90-847a-2524d74343ec"} {"level":"info","ts":1789683761.926941,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683761.9270463,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7"} {"level":"info","ts":1789683761.9280064,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7"} {"level":"info","ts":1789683761.9280329,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7"} {"level":"info","ts":1789683761.9280562,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7","EoS":false} {"level":"info","ts":1789683761.928074,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7"} {"level":"info","ts":1789683761.9281144,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7","EoS":true} {"level":"info","ts":1789683761.928125,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a6a512bd889ceee691fc37ce4c9f2efe","span_id":"8db20a44e30b3a46","x-request-id":"8d1569e2-aa0a-42cc-a9c0-5f881cdfd8b7"} {"level":"info","ts":1789683762.7015574,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683762.7016678,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683762.7016866,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683762.7017019,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","EoS":true} {"level":"info","ts":1789683762.701727,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","header":"x-maas-keyname"} {"level":"info","ts":1789683762.701786,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","header":"x-maas-subscription"} {"level":"info","ts":1789683762.7017922,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","header":"x-maas-username"} {"level":"info","ts":1789683762.7017968,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","header":"x-maas-group"} {"level":"info","ts":1789683762.7018113,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683762.701838,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683762.7026844,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683762.7027042,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683762.702716,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","EoS":false} {"level":"info","ts":1789683762.7027273,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683762.7027583,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc","EoS":true} {"level":"info","ts":1789683762.7027674,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f83dd866d6c84e7bd0f90c15c2766b23","span_id":"a604277ae7adba8d","x-request-id":"bfa3422c-8397-4825-bc7d-267fb7a914dc"} {"level":"info","ts":1789683763.6530676,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683763.6531749,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683763.6531873,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683763.6531985,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","EoS":true} {"level":"info","ts":1789683763.6532245,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","header":"x-maas-subscription"} {"level":"info","ts":1789683763.6532345,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","header":"x-maas-keyname"} {"level":"info","ts":1789683763.6532393,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","header":"x-maas-username"} {"level":"info","ts":1789683763.653244,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","header":"x-maas-group"} {"level":"info","ts":1789683763.6533308,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683763.6533663,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683763.6541872,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683763.6542058,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683763.6542184,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","EoS":false} {"level":"info","ts":1789683763.654229,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683763.654257,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f","EoS":true} {"level":"info","ts":1789683763.6542668,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a015d6502e3a8486ce6876a2970890e7","span_id":"73e528a21eedb758","x-request-id":"ebe14995-e285-49be-b8dd-91c13e36aa9f"} {"level":"info","ts":1789683766.5268943,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683766.5270183,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683766.5270362,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683766.5270553,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","EoS":true} {"level":"info","ts":1789683766.5270941,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","header":"x-maas-username"} {"level":"info","ts":1789683766.5271068,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","header":"x-maas-subscription"} {"level":"info","ts":1789683766.5271158,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","header":"x-maas-group"} {"level":"info","ts":1789683766.5271237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","header":"x-maas-keyname"} {"level":"info","ts":1789683766.5271447,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683766.5271769,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683766.528077,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683766.5280976,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683766.528112,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","EoS":false} {"level":"info","ts":1789683766.5281236,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683766.5281487,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0","EoS":true} {"level":"info","ts":1789683766.5281584,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"958299519258c9a75d082efb2e865b45","span_id":"98dfb5d90b29daed","x-request-id":"2911a3a6-9fa4-4c75-bd65-0346d4d3cce0"} {"level":"info","ts":1789683767.613645,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683767.6137815,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"info","ts":1789683767.6138017,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"info","ts":1789683767.6138222,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","EoS":true} {"level":"info","ts":1789683767.6138613,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","header":"x-maas-username"} {"level":"info","ts":1789683767.6138744,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","header":"x-maas-keyname"} {"level":"info","ts":1789683767.6138818,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","header":"x-maas-subscription"} {"level":"info","ts":1789683767.6138904,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","header":"x-maas-group"} {"level":"info","ts":1789683767.6139123,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683767.6139455,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"info","ts":1789683767.6147954,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"info","ts":1789683767.6148167,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"info","ts":1789683767.6148295,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","EoS":false} {"level":"info","ts":1789683767.6148417,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"info","ts":1789683767.6148698,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e","EoS":true} {"level":"info","ts":1789683767.6148925,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8465c81a71e4c18160d0fbd3f55a209b","span_id":"06e3a26994b6947f","x-request-id":"476db96d-7497-4896-a1d7-ff264ee2857e"} {"level":"error","ts":1789683775.1284466,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683789.551021,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683789.551147,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"info","ts":1789683789.5511632,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"info","ts":1789683789.551184,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","EoS":true} {"level":"info","ts":1789683789.551223,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","header":"x-maas-subscription"} {"level":"info","ts":1789683789.551237,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","header":"x-maas-keyname"} {"level":"info","ts":1789683789.5512462,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","header":"x-maas-username"} {"level":"info","ts":1789683789.5512524,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","header":"x-maas-group"} {"level":"info","ts":1789683789.551273,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683789.5513322,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"info","ts":1789683789.5522296,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"info","ts":1789683789.5522501,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"info","ts":1789683789.5522642,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","EoS":false} {"level":"info","ts":1789683789.552275,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"info","ts":1789683789.552314,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c","EoS":true} {"level":"info","ts":1789683789.552322,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f6e9e08238da0fa395324147ea07fa70","span_id":"81d359ce8420035b","x-request-id":"28c11a78-5de8-40e1-8e53-651eabcb282c"} {"level":"error","ts":1789683790.130381,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789683800.1314256,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683881.0246093,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683881.024715,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"info","ts":1789683881.024732,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"info","ts":1789683881.024747,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","EoS":true} {"level":"info","ts":1789683881.024771,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","header":"x-maas-group"} {"level":"info","ts":1789683881.0247831,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","header":"x-maas-subscription"} {"level":"info","ts":1789683881.024788,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","header":"x-maas-keyname"} {"level":"info","ts":1789683881.0247931,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","header":"x-maas-username"} {"level":"info","ts":1789683881.0248094,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789683881.0248399,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"info","ts":1789683881.0376034,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"info","ts":1789683881.0376284,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"info","ts":1789683881.0376444,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","EoS":false} {"level":"info","ts":1789683881.037656,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"info","ts":1789683881.0376832,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df","EoS":true} {"level":"info","ts":1789683881.0376914,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5fea5c643a865d34af77724ecdd6c4b4","span_id":"3f74014467f411f0","x-request-id":"d2bf7993-133c-4df9-bd65-88f4f51df6df"} {"level":"error","ts":1789683895.1424882,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683948.084557,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683948.084667,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"info","ts":1789683948.08468,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"info","ts":1789683948.0846932,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","EoS":true} {"level":"info","ts":1789683948.0847194,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","header":"x-maas-keyname"} {"level":"info","ts":1789683948.0847294,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","header":"x-maas-group"} {"level":"info","ts":1789683948.0847337,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","header":"x-maas-subscription"} {"level":"info","ts":1789683948.0847387,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","header":"x-maas-username"} {"level":"info","ts":1789683948.0847545,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789683948.0847805,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"info","ts":1789683948.0861747,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"info","ts":1789683948.086195,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"info","ts":1789683948.0862088,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","EoS":false} {"level":"info","ts":1789683948.0862195,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"info","ts":1789683948.086247,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a","EoS":true} {"level":"info","ts":1789683948.0862617,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5add9d58375e6c96adbc7654af14cd65","span_id":"0f1e36e80cc295e6","x-request-id":"80470290-c061-4af0-8898-6299292c259a"} {"level":"error","ts":1789683960.1487293,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683964.147946,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683964.1480508,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.148063,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.1480742,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","EoS":true} {"level":"info","ts":1789683964.1480975,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","header":"x-maas-group"} {"level":"info","ts":1789683964.1481078,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","header":"x-maas-username"} {"level":"info","ts":1789683964.1481125,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","header":"x-maas-subscription"} {"level":"info","ts":1789683964.1481168,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","header":"x-maas-keyname"} {"level":"info","ts":1789683964.1481311,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789683964.1481576,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.1617846,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.1618094,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.1618273,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","EoS":false} {"level":"info","ts":1789683964.1618397,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.161869,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","EoS":false} {"level":"info","ts":1789683964.1618803,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.1618922,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a","EoS":true} {"level":"info","ts":1789683964.161899,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"085242f8916dbfcb3000f329dac414d4","span_id":"66ee345cbfa6e689","x-request-id":"987aee32-ad6d-4bc0-b3a7-b006f429fb0a"} {"level":"info","ts":1789683964.304126,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683964.304228,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.3042414,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.304253,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","EoS":true} {"level":"info","ts":1789683964.3042753,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","header":"x-maas-keyname"} {"level":"info","ts":1789683964.3043115,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","header":"x-maas-group"} {"level":"info","ts":1789683964.3043168,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","header":"x-maas-subscription"} {"level":"info","ts":1789683964.304321,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","header":"x-maas-username"} {"level":"info","ts":1789683964.3043365,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789683964.3043606,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.30698,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.3069944,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.3070097,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","EoS":false} {"level":"info","ts":1789683964.3070219,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.3070445,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","EoS":false} {"level":"info","ts":1789683964.3070562,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.3070667,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2","EoS":true} {"level":"info","ts":1789683964.3070734,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82ea82dfa0b544ff5dc1209b45cfa1a2","span_id":"a2b08cdfbe048d68","x-request-id":"9f6cd07b-7dec-485a-bd2a-4594b1b129a2"} {"level":"info","ts":1789683964.449262,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683964.4494133,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.449429,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.449448,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","EoS":true} {"level":"info","ts":1789683964.4494789,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","header":"x-maas-keyname"} {"level":"info","ts":1789683964.4494948,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","header":"x-maas-group"} {"level":"info","ts":1789683964.4495072,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","header":"x-maas-username"} {"level":"info","ts":1789683964.4495153,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","header":"x-maas-subscription"} {"level":"info","ts":1789683964.4495368,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789683964.4495754,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.4507391,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.4507601,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.4507768,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","EoS":false} {"level":"info","ts":1789683964.4507918,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.4508183,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","EoS":false} {"level":"info","ts":1789683964.4508295,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"info","ts":1789683964.450839,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf","EoS":true} {"level":"info","ts":1789683964.450846,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"edb9d25edf1be3dee1dc269d34c8301d","span_id":"90e7458252745f92","x-request-id":"430a6a2e-e43f-43d2-9ed6-af8787206faf"} {"level":"error","ts":1789683975.1507435,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789683977.3722625,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789683977.3724267,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"info","ts":1789683977.3724499,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"info","ts":1789683977.372471,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","EoS":true} {"level":"info","ts":1789683977.3725038,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","header":"x-maas-subscription"} {"level":"info","ts":1789683977.3725185,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","header":"x-maas-keyname"} {"level":"info","ts":1789683977.3725262,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","header":"x-maas-username"} {"level":"info","ts":1789683977.3725343,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","header":"x-maas-group"} {"level":"info","ts":1789683977.3725488,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789683977.3725767,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"info","ts":1789683977.374649,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"info","ts":1789683977.37467,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"info","ts":1789683977.374686,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","EoS":false} {"level":"info","ts":1789683977.3746982,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"info","ts":1789683977.3747253,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b","EoS":true} {"level":"info","ts":1789683977.3747356,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"91daba94e6f3dddb84410a0fd8465f34","span_id":"9023328a3600fdde","x-request-id":"51dfa069-6d09-9342-a9b0-3536c2dfb61b"} {"level":"error","ts":1789683990.1519408,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}