{"level":"info","ts":1789477125.6844678,"logger":"setup","caller":"cmd/runner.go:103","msg":"ai-gateway-payload-processing build","commit-sha":"unknown","build-ref":""}
{"level":"info","ts":1789477125.6845474,"logger":"setup.trace","caller":"tracing/telemetry.go:131","msg":"init OTel trace exporter","type":"otlp"}
{"level":"info","ts":1789477125.6861064,"logger":"setup","caller":"cmd/runner.go:172","msg":"Setting pprof handlers"}
{"level":"info","ts":1789477125.686126,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/heap"}
{"level":"info","ts":1789477125.686138,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/goroutine"}
{"level":"info","ts":1789477125.686143,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/allocs"}
{"level":"info","ts":1789477125.6861472,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/threadcreate"}
{"level":"info","ts":1789477125.6861515,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/block"}
{"level":"info","ts":1789477125.6861553,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/mutex"}
{"level":"info","ts":1789477125.686896,"logger":"setup","caller":"loader/configloader.go:171","msg":"Loaded raw configuration","config":"{Plugins: [{Name: maas-headers-guard, Type: maas-headers-guard} {Type: stream-usage-enforcer} {Type: model-provider-resolver} {Type: api-translation} {Type: apikey-injection}], Profiles: [{Name: default, Plugins: {Request: [{PluginRef: maas-headers-guard} {PluginRef: stream-usage-enforcer} {PluginRef: model-provider-resolver} {PluginRef: api-translation} {PluginRef: apikey-injection}]}}]}"}
{"level":"info","ts":1789477125.6870217,"caller":"api-translation/plugin.go:70","msg":"plugin initialized","translators":4}
{"level":"info","ts":1789477125.7068396,"logger":"setup","caller":"cmd/runner.go:242","msg":"manager starting"}
{"level":"info","ts":1789477125.7070122,"logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":1789477125.7073193,"logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":9090","secure":false}
{"level":"info","ts":1789477125.7078652,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"health"}
{"level":"info","ts":1789477125.7079291,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"health","port":9005}
{"level":"info","ts":1789477125.7086031,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789477125.7089005,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"apikey-injection-secret-watcher","source":"kind source: *v1.Secret"}
{"level":"info","ts":1789477125.7095053,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789477125.7096093,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1.Secret","resyncPeriod":32857.572669255,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.7096245,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.7098024,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":1789477125.7128346,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.7195735,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalProvider","resyncPeriod":32647.559521269,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.719592,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.7198362,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalModel","resyncPeriod":37760.936428173,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.71985,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.72234,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.7228024,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789477125.9107335,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"apikey-injection-secret-watcher"}
{"level":"info","ts":1789477125.9107816,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"apikey-injection-secret-watcher","worker count":1}
{"level":"info","ts":1789477125.9108655,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"329955a0-56f8-4489-a4aa-11d9367981c8","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789477125.9109106,"caller":"apikey-injection/reconciler.go:71","msg":"Secret added/updated in store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"329955a0-56f8-4489-a4aa-11d9367981c8","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789477125.9210954,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel"}
{"level":"info","ts":1789477125.9211285,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","worker count":1}
{"level":"info","ts":1789477125.9212277,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"c6882ec8-f0cc-43b2-8036-b58644e23642","name":"e2e-external-model","namespace":"llm"}
{"level":"error","ts":1789477125.9212675,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"c6882ec8-f0cc-43b2-8036-b58644e23642","provider":"e2e-external-provider","error":"ExternalProvider \"e2e-external-provider\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789477125.921287,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"c6882ec8-f0cc-43b2-8036-b58644e23642"}
{"level":"info","ts":1789477125.9213517,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider"}
{"level":"info","ts":1789477125.9213676,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","worker count":1}
{"level":"info","ts":1789477125.9214048,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"c9fa4862-d08d-4baa-9023-2b743cc82d6c","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789477125.9214303,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"c9fa4862-d08d-4baa-9023-2b743cc82d6c","provider":"openai","endpoint":"httpbingo.org"}
{"level":"info","ts":1789477126.5720136,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"ext-proc"}
{"level":"info","ts":1789477126.572122,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"ext-proc","port":9004}
{"level":"info","ts":1789477130.921771,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"9216b0d8-e519-4f22-ba76-603c2538cbfc","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789477130.9218385,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"9216b0d8-e519-4f22-ba76-603c2538cbfc","modelName":"e2e-external-model","resolvedRefs":1}
{"level":"info","ts":1789477153.7375453,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"5f173ee9-3754-492a-93c5-0db384eb8324","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789477153.7376351,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"5f173ee9-3754-492a-93c5-0db384eb8324","name":"e2e-external-model","namespace":"llm"}
{"level":"info","ts":1789477153.8700411,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"99bbad2c-0ec1-45e2-b329-47c87a7adfdd","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789477153.870166,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"99bbad2c-0ec1-45e2-b329-47c87a7adfdd","name":"e2e-external-provider","namespace":"llm"}
{"level":"info","ts":1789477153.9795456,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"55bcefc4-0e37-441e-bf3a-1404cea6cb06","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789477153.9795935,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"55bcefc4-0e37-441e-bf3a-1404cea6cb06","key":"llm/e2e-external-model-api-key"}
{"level":"info","ts":1789477470.2896898,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789477470.2902052,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"info","ts":1789477470.2902246,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"info","ts":1789477470.2902594,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","EoS":true}
{"level":"info","ts":1789477470.2902856,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","header":"x-maas-subscription"}
{"level":"info","ts":1789477470.2902954,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","header":"x-maas-group"}
{"level":"info","ts":1789477470.2903006,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","header":"x-maas-username"}
{"level":"info","ts":1789477470.2903235,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","path":"/ai-tenant-e2e-inf-b-1c35dd/test-model-1c35dd/v1/chat/completions"}
{"level":"info","ts":1789477470.290358,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"info","ts":1789477470.3035083,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"info","ts":1789477470.3035376,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"info","ts":1789477470.3035567,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","EoS":false}
{"level":"info","ts":1789477470.3035705,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"info","ts":1789477470.303598,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952","EoS":true}
{"level":"info","ts":1789477470.3036058,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e229edb85f5a5f57e5eefbfea613370b","span_id":"a7b8c401bb173e5c","x-request-id":"6c12b181-694a-4578-a551-67dad040a952"}
{"level":"error","ts":1789477480.7198553,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}