{"level":"info","ts":1789748923.3075342,"logger":"setup","caller":"cmd/runner.go:103","msg":"ai-gateway-payload-processing build","commit-sha":"unknown","build-ref":""}
{"level":"info","ts":1789748923.3076317,"logger":"setup.trace","caller":"tracing/telemetry.go:131","msg":"init OTel trace exporter","type":"otlp"}
{"level":"info","ts":1789748923.3098202,"logger":"setup","caller":"cmd/runner.go:172","msg":"Setting pprof handlers"}
{"level":"info","ts":1789748923.3098469,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/heap"}
{"level":"info","ts":1789748923.3098636,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/goroutine"}
{"level":"info","ts":1789748923.3098788,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/allocs"}
{"level":"info","ts":1789748923.3098874,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/threadcreate"}
{"level":"info","ts":1789748923.3098953,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/block"}
{"level":"info","ts":1789748923.309903,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/mutex"}
{"level":"info","ts":1789748923.3106523,"logger":"setup","caller":"loader/configloader.go:171","msg":"Loaded raw configuration","config":"{Plugins: [{Name: maas-headers-guard, Type: maas-headers-guard} {Type: stream-usage-enforcer} {Type: model-provider-resolver} {Type: api-translation} {Type: apikey-injection}], Profiles: [{Name: default, Plugins: {Request: [{PluginRef: maas-headers-guard} {PluginRef: stream-usage-enforcer} {PluginRef: model-provider-resolver} {PluginRef: api-translation} {PluginRef: apikey-injection}]}}]}"}
{"level":"info","ts":1789748923.3108265,"caller":"api-translation/plugin.go:70","msg":"plugin initialized","translators":4}
{"level":"info","ts":1789748923.3366916,"logger":"setup","caller":"cmd/runner.go:242","msg":"manager starting"}
{"level":"info","ts":1789748923.3367305,"logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":1789748923.3370025,"logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":9090","secure":false}
{"level":"info","ts":1789748923.337583,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"health"}
{"level":"info","ts":1789748923.337947,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789748923.3382347,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"apikey-injection-secret-watcher","source":"kind source: *v1.Secret"}
{"level":"info","ts":1789748923.3384387,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":1789748923.338519,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1.Secret","resyncPeriod":39495.189650256,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.338534,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.3386943,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":1789748923.338888,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"health","port":9005}
{"level":"info","ts":1789748923.3428652,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.3495288,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalProvider","resyncPeriod":37502.063302041,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.349552,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.3498585,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalModel","resyncPeriod":36519.084461007,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.3498755,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.386563,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.387162,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789748923.4393928,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"apikey-injection-secret-watcher"}
{"level":"info","ts":1789748923.439419,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"apikey-injection-secret-watcher","worker count":1}
{"level":"info","ts":1789748923.4495509,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider"}
{"level":"info","ts":1789748923.4495819,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","worker count":1}
{"level":"info","ts":1789748923.4506981,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel"}
{"level":"info","ts":1789748923.450719,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","worker count":1}
{"level":"info","ts":1789748923.997078,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"ext-proc"}
{"level":"info","ts":1789748923.997174,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"ext-proc","port":9004}
{"level":"info","ts":1789750169.6541252,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789750169.657114,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"}
{"level":"info","ts":1789750278.2063096,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d7502777-0260-4b6e-923c-cf8e53048216","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789750278.2063665,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d7502777-0260-4b6e-923c-cf8e53048216","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789750278.2063942,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d7502777-0260-4b6e-923c-cf8e53048216"}
{"level":"info","ts":1789750278.217176,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"96cf8c98-61b4-4adb-9b7e-8af0984fdff8","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789750278.217293,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"96cf8c98-61b4-4adb-9b7e-8af0984fdff8","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789750278.2173471,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"96cf8c98-61b4-4adb-9b7e-8af0984fdff8"}
{"level":"info","ts":1789750278.232788,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6204fe34-58ed-4a3a-999f-75a7c495191b","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789750278.2328215,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6204fe34-58ed-4a3a-999f-75a7c495191b","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789750278.2328424,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6204fe34-58ed-4a3a-999f-75a7c495191b"}
{"level":"info","ts":1789750278.3552713,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8bf55064-955a-4f4a-ac32-cfe1decaeeb9","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789750278.3553178,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8bf55064-955a-4f4a-ac32-cfe1decaeeb9","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789750278.3553457,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8bf55064-955a-4f4a-ac32-cfe1decaeeb9"}
{"level":"info","ts":1789750283.3565392,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4ff0f0ae-5dd0-410e-971b-df2ff4f12b7c","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789750283.3565826,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4ff0f0ae-5dd0-410e-971b-df2ff4f12b7c","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789750283.3566027,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4ff0f0ae-5dd0-410e-971b-df2ff4f12b7c"}
{"level":"info","ts":1789750283.9815953,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4551bd5f-792d-44c0-a7c3-a52ad3e2dd9e","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"info","ts":1789750283.981648,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4551bd5f-792d-44c0-a7c3-a52ad3e2dd9e","name":"e2e-x-api-key-trigger","namespace":"llm"}