{"level":"info","ts":1789480340.8952062,"logger":"setup","caller":"cmd/runner.go:103","msg":"ai-gateway-payload-processing build","commit-sha":"unknown","build-ref":""}
{"level":"info","ts":1789480340.8952994,"logger":"setup.trace","caller":"tracing/telemetry.go:131","msg":"init OTel trace exporter","type":"otlp"}
{"level":"info","ts":1789480340.8983479,"logger":"setup","caller":"cmd/runner.go:172","msg":"Setting pprof handlers"}
{"level":"info","ts":1789480340.8983831,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/heap"}
{"level":"info","ts":1789480340.8983972,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/goroutine"}
{"level":"info","ts":1789480340.8984025,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/allocs"}
{"level":"info","ts":1789480340.8984075,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/threadcreate"}
{"level":"info","ts":1789480340.8984122,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/block"}
{"level":"info","ts":1789480340.8984158,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/mutex"}
{"level":"info","ts":1789480340.8989563,"logger":"setup","caller":"loader/configloader.go:171","msg":"Loaded raw configuration","config":"{Plugins: [{Name: maas-headers-guard, Type: maas-headers-guard} {Type: stream-usage-enforcer} {Type: model-provider-resolver} {Type: api-translation} {Type: apikey-injection}], Profiles: [{Name: default, Plugins: {Request: [{PluginRef: maas-headers-guard} {PluginRef: stream-usage-enforcer} {PluginRef: model-provider-resolver} {PluginRef: api-translation} {PluginRef: apikey-injection}]}}]}"}
{"level":"info","ts":1789480340.899107,"caller":"api-translation/plugin.go:70","msg":"plugin initialized","translators":4}
{"level":"info","ts":1789480340.9240205,"logger":"setup","caller":"cmd/runner.go:242","msg":"manager starting"}
{"level":"info","ts":1789480340.9240758,"logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":1789480340.9244125,"logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":9090","secure":false}
{"level":"info","ts":1789480340.924504,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"health"}
{"level":"info","ts":1789480340.9245656,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"health","port":9005}
{"level":"info","ts":1789480340.924884,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789480340.9253087,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"apikey-injection-secret-watcher","source":"kind source: *v1.Secret"}
{"level":"info","ts":1789480340.9256475,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789480340.9258053,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1.Secret","resyncPeriod":39432.491460434,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.9258223,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.926031,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":1789480340.9290295,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.936311,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalProvider","resyncPeriod":37651.853854389,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.9363337,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.9366214,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalModel","resyncPeriod":38207.810277512,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.936638,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.9398682,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480340.9401584,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789480341.1267722,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"apikey-injection-secret-watcher"}
{"level":"info","ts":1789480341.1268454,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"apikey-injection-secret-watcher","worker count":1}
{"level":"info","ts":1789480341.1269603,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"2bba4358-b39b-4ab5-92c0-c3520824752c","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789480341.12702,"caller":"apikey-injection/reconciler.go:71","msg":"Secret added/updated in store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"2bba4358-b39b-4ab5-92c0-c3520824752c","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789480341.1372404,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider"}
{"level":"info","ts":1789480341.1372998,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","worker count":1}
{"level":"info","ts":1789480341.1374657,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"8a7ec756-8130-4aa8-9c6d-4384a94d46d7","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789480341.1375263,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"8a7ec756-8130-4aa8-9c6d-4384a94d46d7","provider":"openai","endpoint":"httpbingo.org"}
{"level":"info","ts":1789480341.1376016,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel"}
{"level":"info","ts":1789480341.1376293,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","worker count":1}
{"level":"info","ts":1789480341.137715,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"0a9c6a56-9b70-4408-a4d7-b64a24112314","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789480341.1377559,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"0a9c6a56-9b70-4408-a4d7-b64a24112314","modelName":"e2e-external-model","resolvedRefs":1}
{"level":"info","ts":1789480341.7916377,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"ext-proc"}
{"level":"info","ts":1789480341.791755,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"ext-proc","port":9004}
{"level":"info","ts":1789480376.9504378,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"d199ea97-f6cb-43b8-a7fb-07d46810ce1e","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789480376.9504967,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"d199ea97-f6cb-43b8-a7fb-07d46810ce1e","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789480377.0875764,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"d0f04502-86bd-4802-9512-d03d42f9be11","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789480377.0876007,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"d0f04502-86bd-4802-9512-d03d42f9be11","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789480377.194758,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"21d1a853-9b5a-484f-ba46-b4548a14072b","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789480377.1948965,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"21d1a853-9b5a-484f-ba46-b4548a14072b","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789480646.1180995,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789480646.1185844,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"info","ts":1789480646.118604,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"info","ts":1789480646.118635,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","EoS":true}
{"level":"info","ts":1789480646.118693,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","header":"x-maas-username"}
{"level":"info","ts":1789480646.1187067,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","header":"x-maas-subscription"}
{"level":"info","ts":1789480646.1187115,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","header":"x-maas-group"}
{"level":"info","ts":1789480646.1187336,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","path":"/ai-tenant-e2e-inf-b-2aa332/test-model-2aa332/v1/chat/completions"}
{"level":"info","ts":1789480646.1187675,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"info","ts":1789480646.132782,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"info","ts":1789480646.1328106,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"info","ts":1789480646.1328313,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","EoS":false}
{"level":"info","ts":1789480646.13285,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"info","ts":1789480646.132877,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9","EoS":true}
{"level":"info","ts":1789480646.1328864,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4e22ec6d4e6d70d27602c63f01afe8a2","span_id":"4e223a6bf6608ba7","x-request-id":"392cb4ad-280a-4c60-b2a7-2d7e678348b9"}
{"level":"error","ts":1789480660.92987,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}