{"level":"info","ts":1788927044.1406736,"logger":"setup","caller":"cmd/runner.go:103","msg":"ai-gateway-payload-processing build","commit-sha":"unknown","build-ref":""} {"level":"info","ts":1788927044.140777,"logger":"setup.trace","caller":"tracing/telemetry.go:131","msg":"init OTel trace exporter","type":"otlp"} {"level":"info","ts":1788927044.1430788,"logger":"setup","caller":"cmd/runner.go:172","msg":"Setting pprof handlers"} {"level":"info","ts":1788927044.1431527,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/heap"} {"level":"info","ts":1788927044.143182,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/goroutine"} {"level":"info","ts":1788927044.143194,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/allocs"} {"level":"info","ts":1788927044.1432078,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/threadcreate"} {"level":"info","ts":1788927044.1432185,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/block"} {"level":"info","ts":1788927044.1432323,"caller":"manager/internal.go:201","msg":"Registering metrics http server extra handler","path":"/debug/pprof/mutex"} {"level":"info","ts":1788927044.14432,"logger":"setup","caller":"loader/configloader.go:171","msg":"Loaded raw configuration","config":"{Plugins: [{Name: maas-headers-guard, Type: maas-headers-guard} {Type: stream-usage-enforcer} {Type: model-provider-resolver} {Type: api-translation} {Type: apikey-injection}], Profiles: [{Name: default, Plugins: {Request: [{PluginRef: maas-headers-guard} {PluginRef: stream-usage-enforcer} {PluginRef: model-provider-resolver} {PluginRef: api-translation} {PluginRef: apikey-injection}]}}]}"} {"level":"info","ts":1788927044.1444764,"caller":"api-translation/plugin.go:70","msg":"plugin initialized","translators":4} {"level":"info","ts":1788927044.1666493,"logger":"setup","caller":"cmd/runner.go:242","msg":"manager starting"} {"level":"info","ts":1788927044.1668046,"logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"} {"level":"info","ts":1788927044.167231,"logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":9090","secure":false} {"level":"info","ts":1788927044.1673262,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"health"} {"level":"info","ts":1788927044.1674075,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"health","port":9005} {"level":"info","ts":1788927044.1676378,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1alpha1.ExternalProvider"} {"level":"info","ts":1788927044.167953,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"apikey-injection-secret-watcher","source":"kind source: *v1.Secret"} {"level":"info","ts":1788927044.1682043,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalProvider"} {"level":"info","ts":1788927044.168323,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1.Secret","resyncPeriod":32467.250776328,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.1683342,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.1684835,"caller":"controller/controller.go:370","msg":"Starting EventSource","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"} {"level":"info","ts":1788927044.1718528,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1.Secret","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.182654,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalProvider","resyncPeriod":33762.331451108,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.182677,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.1832008,"logger":"controller-runtime.cache","caller":"cache/reflector.go:367","msg":"Starting reflector","type":"*v1alpha1.ExternalModel","resyncPeriod":37780.116215135,"reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.1832185,"logger":"controller-runtime.cache","caller":"cache/reflector.go:411","msg":"Listing and watching","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.1857858,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalProvider","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.1865532,"logger":"controller-runtime.cache","caller":"cache/reflector.go:446","msg":"Caches populated","type":"*v1alpha1.ExternalModel","reflector":"k8s.io/client-go@v0.35.7/tools/cache/reflector.go:289"} {"level":"info","ts":1788927044.2838426,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel"} {"level":"info","ts":1788927044.2838817,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","worker count":1} {"level":"info","ts":1788927044.2840335,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"b2ac1e3c-a360-44f5-a9e8-ad37ebb4f1cc","name":"e2e-external-model","namespace":"llm"} {"level":"error","ts":1788927044.2840722,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"b2ac1e3c-a360-44f5-a9e8-ad37ebb4f1cc","provider":"e2e-external-provider","error":"ExternalProvider \"e2e-external-provider\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1788927044.284107,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"b2ac1e3c-a360-44f5-a9e8-ad37ebb4f1cc"} {"level":"info","ts":1788927044.2841747,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"1fcc6e46-0c0d-4472-aa22-45871537fb86","name":"e2e-cleanup-test","namespace":"llm"} {"level":"error","ts":1788927044.2841947,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"1fcc6e46-0c0d-4472-aa22-45871537fb86","provider":"e2e-external-provider","error":"ExternalProvider \"e2e-external-provider\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1788927044.2842155,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"1fcc6e46-0c0d-4472-aa22-45871537fb86"} {"level":"info","ts":1788927044.3695338,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"apikey-injection-secret-watcher"} {"level":"info","ts":1788927044.369589,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"apikey-injection-secret-watcher","worker count":1} {"level":"info","ts":1788927044.3696804,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"73288fb3-c781-4c4c-ad1c-40507d8d109b","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1788927044.36974,"caller":"apikey-injection/reconciler.go:71","msg":"Secret added/updated in store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"73288fb3-c781-4c4c-ad1c-40507d8d109b","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1788927044.3849556,"caller":"controller/controller.go:303","msg":"Starting Controller","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider"} {"level":"info","ts":1788927044.3850012,"caller":"controller/controller.go:306","msg":"Starting workers","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","worker count":1} {"level":"info","ts":1788927044.3850636,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"136807ab-e1d3-440a-9b3a-e9efe100c5f1","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1788927044.385149,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"136807ab-e1d3-440a-9b3a-e9efe100c5f1","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1788927045.5982962,"caller":"cmd/runner.go:316","msg":"gRPC server starting","name":"ext-proc"} {"level":"info","ts":1788927045.5984666,"caller":"cmd/runner.go:323","msg":"gRPC server listening","name":"ext-proc","port":9004} {"level":"info","ts":1788927049.2862852,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"b1eb821f-e9a0-4553-b00f-0aa4a9607d8e","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1788927049.2863798,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"b1eb821f-e9a0-4553-b00f-0aa4a9607d8e","modelName":"e2e-external-model","resolvedRefs":1} {"level":"info","ts":1788927049.2864351,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"b23be4f8-9624-4dea-aa53-a1edbb9d13d3","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1788927049.286451,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"b23be4f8-9624-4dea-aa53-a1edbb9d13d3","modelName":"e2e-cleanup-test","resolvedRefs":1} {"level":"info","ts":1788927051.5898309,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"9a053089-9242-474f-8771-f3ca0bbc057e","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1788927051.589878,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cleanup-test","namespace":"llm"},"namespace":"llm","name":"e2e-cleanup-test","reconcileID":"9a053089-9242-474f-8771-f3ca0bbc057e","name":"e2e-cleanup-test","namespace":"llm"} {"level":"info","ts":1788927056.2675433,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"dc4072e1-b990-49ba-9c80-6115de7e4a8c","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.267586,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"dc4072e1-b990-49ba-9c80-6115de7e4a8c","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1788927056.2753327,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"f07efd88-b4cd-41e1-b49c-e4f53692403b","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.2753634,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"f07efd88-b4cd-41e1-b49c-e4f53692403b","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1788927056.2779038,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"98fcf633-eb60-4e8c-8b63-bb620b77e331","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.2780035,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"98fcf633-eb60-4e8c-8b63-bb620b77e331","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927056.2855783,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"5ff65172-4b18-4c0e-83fc-c203df45460b","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.2856607,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"5ff65172-4b18-4c0e-83fc-c203df45460b","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927056.2916028,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"437c5b7f-9512-4d6a-8884-bdeec1e37f8a","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.2916863,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"437c5b7f-9512-4d6a-8884-bdeec1e37f8a","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927056.3090441,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"39382b2b-9a41-4ba5-8552-55b0e2921cbc","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.3090696,"caller":"model-provider-resolver/external_provider_reconciler.go:77","msg":"updated provider store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"39382b2b-9a41-4ba5-8552-55b0e2921cbc","provider":"openai","endpoint":"httpbingo.org"} {"level":"info","ts":1788927056.309186,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"a49df423-9c75-4b94-9bbc-5ca7e48c2568","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.3092086,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"a49df423-9c75-4b94-9bbc-5ca7e48c2568","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927056.3320618,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"369b66be-ab81-4404-aacf-82ebf9f02d88","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.3321762,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"369b66be-ab81-4404-aacf-82ebf9f02d88","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927056.342883,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"5a434e7f-602b-4c36-8d85-d3cf78fe3098","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.3429399,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"5a434e7f-602b-4c36-8d85-d3cf78fe3098","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927056.3562398,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"40544a4e-58a0-4fd7-85b1-6b26a73244b9","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927056.3562727,"caller":"model-provider-resolver/external_model_reconciler.go:89","msg":"updated model store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"40544a4e-58a0-4fd7-85b1-6b26a73244b9","modelName":"e2e-legacy-migration-48cd84a4","resolvedRefs":1} {"level":"info","ts":1788927059.1867535,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"7dcf56b7-7d49-4c52-bbf1-6da202b17894","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927059.1867917,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"7dcf56b7-7d49-4c52-bbf1-6da202b17894","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927059.1868641,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"b5576b74-b1c4-4315-a359-d5d070eff07b","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"error","ts":1788927059.1868842,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"b5576b74-b1c4-4315-a359-d5d070eff07b","provider":"e2e-legacy-migration-48cd84a4","error":"ExternalProvider \"e2e-legacy-migration-48cd84a4\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1788927059.1869118,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"b5576b74-b1c4-4315-a359-d5d070eff07b"} {"level":"info","ts":1788927059.1885664,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"daedb343-935f-488f-8ed3-03b39cbc7e2b","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927059.1885986,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-legacy-migration-48cd84a4","namespace":"llm"},"namespace":"llm","name":"e2e-legacy-migration-48cd84a4","reconcileID":"daedb343-935f-488f-8ed3-03b39cbc7e2b","name":"e2e-legacy-migration-48cd84a4","namespace":"llm"} {"level":"info","ts":1788927060.6777062,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"8090532d-eeab-45cd-914d-bf7ee72c4ed7","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1788927060.6777987,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-external-model","namespace":"llm"},"namespace":"llm","name":"e2e-external-model","reconcileID":"8090532d-eeab-45cd-914d-bf7ee72c4ed7","name":"e2e-external-model","namespace":"llm"} {"level":"info","ts":1788927060.8129623,"caller":"model-provider-resolver/external_provider_reconciler.go:52","msg":"reconciling ExternalProvider","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"62b8cfdc-88bd-42aa-8552-eaeb02c54763","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1788927060.813057,"caller":"model-provider-resolver/external_provider_reconciler.go:62","msg":"ExternalProvider removed from store","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-external-provider","namespace":"llm"},"namespace":"llm","name":"e2e-external-provider","reconcileID":"62b8cfdc-88bd-42aa-8552-eaeb02c54763","name":"e2e-external-provider","namespace":"llm"} {"level":"info","ts":1788927060.9265418,"caller":"apikey-injection/reconciler.go:53","msg":"reconciling Secret","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"881d733f-41bd-4982-8db1-b6c5b272c3be","key":"llm/e2e-external-model-api-key"} {"level":"info","ts":1788927060.9266367,"caller":"apikey-injection/reconciler.go:63","msg":"Secret removed from store","controller":"apikey-injection-secret-watcher","namespace":"llm","name":"e2e-external-model-api-key","reconcileID":"881d733f-41bd-4982-8db1-b6c5b272c3be","key":"llm/e2e-external-model-api-key"}