{"level":"info","ts":1789564886.1864557,"logger":"setup","caller":"cmd/runner.go:103","msg":"ai-gateway-payload-processing build","commit-sha":"unknown","build-ref":""}
{"level":"info","ts":1789564886.1865609,"logger":"setup.trace","caller":"tracing/telemetry.go:131","msg":"init OTel trace exporter","type":"otlp"}
{"level":"info","ts":1789564886.1905205,"logger":"setup","caller":"cmd/runner.go:172","msg":"Setting pprof handlers"}
{"level":"info","ts":1789564886.1905475,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/heap"}
{"level":"info","ts":1789564886.190561,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/goroutine"}
{"level":"info","ts":1789564886.1905713,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/allocs"}
{"level":"info","ts":1789564886.1905756,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/threadcreate"}
{"level":"info","ts":1789564886.1905808,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/block"}
{"level":"info","ts":1789564886.1905847,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/mutex"}
{"level":"info","ts":1789564886.19133,"logger":"setup","caller":"loader/configloader.go:171","msg":"Loaded raw configuration","config":"{Plugins: [{Name: maas-headers-guard, Type: maas-headers-guard} {Type: stream-usage-enforcer} {Type: model-provider-resolver} {Type: api-translation} {Type: apikey-injection}], Profiles: [{Name: default, Plugins: {Request: [{PluginRef: maas-headers-guard} {PluginRef: stream-usage-enforcer} {PluginRef: model-provider-resolver} {PluginRef: api-translation} {PluginRef: apikey-injection}]}}]}"}
{"level":"info","ts":1789564886.1914735,"caller":"api-translation/plugin.go:70","msg":"plugin initialized","translators":4}
{"level":"info","ts":1789564886.2147374,"logger":"setup","caller":"cmd/runner.go:242","msg":"manager starting"}
{"level":"info","ts":1789564886.2147741,"logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":1789564886.215442,"logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":9090","secure":false}
{"level":"info","ts":1789564886.2155023,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"health"}
{"level":"info","ts":1789564886.215559,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"health","port":9005}
{"level":"info","ts":1789564886.215738,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789564886.2159615,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"apikey-injection-secret-watcher","source":"kind source: *v1.Secret"}
{"level":"info","ts":1789564886.2161372,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789564886.2162151,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1.Secret","resyncPeriod":38733.898577568,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.216227,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.216363,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":1789564886.2193995,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.2258303,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalProvider","resyncPeriod":35235.520344997,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.2258484,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.226104,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalModel","resyncPeriod":38397.481742526,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.2261193,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.2861285,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.2864742,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789564886.417315,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"apikey-injection-secret-watcher"}
{"level":"info","ts":1789564886.4173539,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"apikey-injection-secret-watcher","worker count":1}
{"level":"info","ts":1789564886.417423,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"1d7061a2-a11e-4e95-9725-e81a56d2851f","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789564886.4174666,"caller":"apikey-injection/reconciler.go:71","msg":"Secret added/updated in store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"1d7061a2-a11e-4e95-9725-e81a56d2851f","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789564886.4266443,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider"}
{"level":"info","ts":1789564886.4266806,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","worker count":1}
{"level":"info","ts":1789564886.4267764,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"e1c93e80-49ea-4aa4-b3b2-ab4e3372abc3","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789564886.426812,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"e1c93e80-49ea-4aa4-b3b2-ab4e3372abc3","provider":"openai","endpoint":"httpbingo.org"}
{"level":"info","ts":1789564886.427921,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel"}
{"level":"info","ts":1789564886.4279392,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","worker count":1}
{"level":"info","ts":1789564886.4279675,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"876c3ffa-973c-42c2-951c-13f005e25a73","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789564886.4279943,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"876c3ffa-973c-42c2-951c-13f005e25a73","modelName":"e2e-external-model","resolvedRefs":1}
{"level":"info","ts":1789564886.4378762,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"ext-proc"}
{"level":"info","ts":1789564886.4379585,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"ext-proc","port":9004}
{"level":"info","ts":1789564918.6968827,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"6d1287d0-7328-4b37-8b6a-9b0de5d716ea","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789564918.6969275,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"6d1287d0-7328-4b37-8b6a-9b0de5d716ea","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789564918.8513238,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"2f9e5b72-3501-441b-97eb-e1d75757637d","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789564918.8513536,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"2f9e5b72-3501-441b-97eb-e1d75757637d","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789564918.9721227,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7cb58075-e5d4-4973-8097-4e4af0324d0c","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789564918.972159,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"7cb58075-e5d4-4973-8097-4e4af0324d0c","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789565196.672349,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789565196.6729226,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"info","ts":1789565196.6729448,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"info","ts":1789565196.6729863,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","EoS":true}
{"level":"info","ts":1789565196.673023,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","header":"x-maas-group"}
{"level":"info","ts":1789565196.673035,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","header":"x-maas-subscription"}
{"level":"info","ts":1789565196.673044,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","header":"x-maas-username"}
{"level":"info","ts":1789565196.6730723,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","path":"/ai-tenant-e2e-inf-b-c71ad7/test-model-c71ad7/v1/chat/completions"}
{"level":"info","ts":1789565196.6731133,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"info","ts":1789565196.686539,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"info","ts":1789565196.6865578,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"info","ts":1789565196.6865711,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","EoS":false}
{"level":"info","ts":1789565196.6865845,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"info","ts":1789565196.686627,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e","EoS":true}
{"level":"info","ts":1789565196.686638,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d7c55dd53a654db2d6e13b596fa623f0","span_id":"4574549908c43a58","x-request-id":"2297de67-2bcb-4000-9414-6ef82a3e3c3e"}
{"level":"error","ts":1789565211.221169,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}