{"level":"info","ts":"2026-07-10T22:55:06Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"1f38d9ef-198d-46d0-94f5-104db581372e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:07Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"e8c243b0-f532-4056-8c4f-5e463ae879b1","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"db175a7e-1315-4c37-bb6c-242fe2ee46d2","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"769db9eb-a951-4e07-8df4-4e32673922fd","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:55:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:55:39Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T22:55:39Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7317d8f2-db96-4a0f-b844-ccf537ba1d7b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:55:39Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7317d8f2-db96-4a0f-b844-ccf537ba1d7b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:55:40Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"46ecd6bc-a7b1-4d56-833b-e0c62e032d5b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:40Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"9cc48bd1-6b58-41ca-ae58-eb30e33469a7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"2f2b11b7-a283-44b5-bf35-b37596a777b2","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:44Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"e9f380e2-06f5-4242-af05-b74c5018fe08","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:45Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"fa8a8e2e-0484-4194-8dde-761682369703","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"error","ts":"2026-07-10T22:55:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"411574f1-2ae1-4f82-884b-677031f30b6f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:55:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"411574f1-2ae1-4f82-884b-677031f30b6f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:55:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"30d7976a-3cd4-48a7-b31d-a2fe67589ea5","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"ed470a61-ce66-4ea5-8493-d6dac7d1232a","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"7989bb14-3f5c-43d3-b6ed-edb91bf80d1c","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"5841eadb-8663-486e-b440-705838a95287","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-727180d9","tenantID":"e2e-mt-727180d9","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"5841eadb-8663-486e-b440-705838a95287"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"5841eadb-8663-486e-b440-705838a95287","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"e2e-mt-727180d9-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f9c3ba1e-6bdb-4f09-820f-cf060a00fbbf","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"1bd2a377-9964-4a84-a759-6f391bf1873b","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"error","ts":"2026-07-10T22:55:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"8acb8c33-c480-442b-bc69-0f8bcb36cdf7","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-sub-727180d9\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3dcf05e5-8234-44ba-9937-d5e84da88a5a","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"4bd8645d-692f-4d2a-9994-b4a2855b8cb9","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-727180d9","tenantID":"e2e-mt-727180d9","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"6b4bd36c-c89b-44ad-9ea5-3338a2b9a333","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"4bd8645d-692f-4d2a-9994-b4a2855b8cb9"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"4bd8645d-692f-4d2a-9994-b4a2855b8cb9","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"e2e-mt-727180d9-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"d4dd26b4-545d-4b10-800f-32fc18e14c70","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","model":"e2e-lifecycle-model-727180d9","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-mt-727180d9/e2e-sub-727180d9"]}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"0f1f2e84-422e-43d2-a688-ca3c61c97038","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"96c37421-d706-43a1-a1c7-99a29ec63d60","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:55:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:55:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"01cef81b-c607-4910-a98e-520e8dbb3f61","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:57Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:55:57Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"70a376a2-b68c-4bd8-b70b-9f24b8b38e47","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-727180d9","tenantID":"e2e-mt-727180d9","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:55:57Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"2d2b1d8c-db64-4dc6-8307-cce70992c453","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:57Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"70a376a2-b68c-4bd8-b70b-9f24b8b38e47"}
{"level":"info","ts":"2026-07-10T22:55:57Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"70a376a2-b68c-4bd8-b70b-9f24b8b38e47","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"e2e-mt-727180d9-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:55:58Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"21a95ee8-46a0-49cc-8428-81bcab3f26a5","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"e3ec23f2-d36e-4b84-bc54-53293c01627e","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:55:58Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"59d388ca-adce-466d-94b5-2316e75bf136","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:55:58Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d138f1f1-1b6b-422e-a439-e0bc6ca8caf2","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"error","ts":"2026-07-10T22:56:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"6e399e35-0ef0-43dd-b9d1-29b9ae71c852","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"6e399e35-0ef0-43dd-b9d1-29b9ae71c852","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"80bf5cca-e50f-4caa-aa4a-dabe115ecd52","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-727180d9","tenantID":"e2e-mt-727180d9","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"f53970d1-b68b-42eb-af2a-ac270dd2c1f4","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","namespace":"ai-tenant-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Deleting MaaSAuthPolicy","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","count":1}
{"level":"info","ts":"2026-07-10T22:56:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-727180d9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-727180d9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-727180d9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-727180d9"},"tenantIdentifier":"e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","tenant":"e2e-mt-727180d9","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-10T22:56:02Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-727180d9","deploymentName":"maas-api-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"d78a9a82-454c-43c7-8cea-32e5832910d5","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","namespace":"ai-tenant-e2e-mt-727180d9","llmisvcName":"e2e-lifecycle-model-727180d9","gateway":"openshift-ingress/e2e-mt-727180d9","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"80bf5cca-e50f-4caa-aa4a-dabe115ecd52"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"80bf5cca-e50f-4caa-aa4a-dabe115ecd52","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"e2e-mt-727180d9-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","gvk":"Deployment","name":"maas-api-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"8956a08a-ba8f-4bcb-a781-013ad73743a7","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","gvk":"Service","name":"maas-api-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"default-tenant","reconcileID":"db73525c-0823-4d70-a41c-070a331edce5","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:02Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-policy-727180d9","reconcileID":"5a2c9355-96a8-4170-9371-e54a1ed73944","MaaSAuthPolicy":{"name":"e2e-policy-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:03Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"1a7b2a77-50e3-4dda-90c6-0311c823f9c6","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9","subscription":"e2e-sub-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:03Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"1a7b2a77-50e3-4dda-90c6-0311c823f9c6","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:03Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-sub-727180d9","reconcileID":"1a7b2a77-50e3-4dda-90c6-0311c823f9c6","MaaSSubscription":{"name":"e2e-sub-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"name":"maas-trlp-e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9","model":"ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9"}
{"level":"info","ts":"2026-07-10T22:56:03Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ae9adf7b-072f-434f-b2b2-0549269fcfea","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"497b6287-293c-49f8-ac58-1085493e7b24","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"77aeda44-f2a9-4ace-9ecb-7751f483ec03","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"77aeda44-f2a9-4ace-9ecb-7751f483ec03","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"77aeda44-f2a9-4ace-9ecb-7751f483ec03","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f17050bf-efd1-41f1-b1d9-1c5cf20c37ab","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f17050bf-efd1-41f1-b1d9-1c5cf20c37ab","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f17050bf-efd1-41f1-b1d9-1c5cf20c37ab","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"372a77d2-762d-4ff7-9e4c-545de94c3966","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"372a77d2-762d-4ff7-9e4c-545de94c3966","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"372a77d2-762d-4ff7-9e4c-545de94c3966","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"ec098e87-b2b8-4bd0-9839-855ab28c790f","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"ec098e87-b2b8-4bd0-9839-855ab28c790f","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"ec098e87-b2b8-4bd0-9839-855ab28c790f","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3896662c-0359-47c5-b376-7d34fc70d846","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3896662c-0359-47c5-b376-7d34fc70d846","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3896662c-0359-47c5-b376-7d34fc70d846","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"9993df54-53f0-48db-bb37-660631e17d3a","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"9993df54-53f0-48db-bb37-660631e17d3a","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"9993df54-53f0-48db-bb37-660631e17d3a","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3ad92c2f-57ca-42fe-ad45-1a89652c8c2b","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3ad92c2f-57ca-42fe-ad45-1a89652c8c2b","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"3ad92c2f-57ca-42fe-ad45-1a89652c8c2b","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"fa8d503d-1c19-433c-ada6-3c35175e891a","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"fa8d503d-1c19-433c-ada6-3c35175e891a","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:04Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"fa8d503d-1c19-433c-ada6-3c35175e891a","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:05Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"4ce86b98-5538-4306-b481-6213d7f2d344","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:05Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"4ce86b98-5538-4306-b481-6213d7f2d344","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:05Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"4ce86b98-5538-4306-b481-6213d7f2d344","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:06Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"490d8f80-c18f-4760-92d8-f7cf4cfcbc34","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:06Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"490d8f80-c18f-4760-92d8-f7cf4cfcbc34","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:06Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"490d8f80-c18f-4760-92d8-f7cf4cfcbc34","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:08Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"62ebe4ef-3b1b-4ff4-b427-ef77472e9469","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:08Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"68a96ddb-c5e8-497f-917b-6623724b8646","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"48b1d8d6-07f1-49a0-970e-a7770e2b5219","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"48b1d8d6-07f1-49a0-970e-a7770e2b5219","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"48b1d8d6-07f1-49a0-970e-a7770e2b5219","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"bfafa296-b1fd-460c-b9eb-776b3b66ce05","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"bfafa296-b1fd-460c-b9eb-776b3b66ce05","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"bfafa296-b1fd-460c-b9eb-776b3b66ce05","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f6fba675-3506-464f-8b8f-32705f123cd4","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f6fba675-3506-464f-8b8f-32705f123cd4","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:09Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"f6fba675-3506-464f-8b8f-32705f123cd4","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:13Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"0f8b6f1a-8581-4552-999d-7767a338540b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:14Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"8a355424-f821-4335-9516-b212e4a06970","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-10T22:56:19Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"bfd93c96-fdc4-4628-959a-d104daf2e859","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"routeName":"e2e-lifecycle-model-727180d9-kserve-route","routeNamespace":"ai-tenant-e2e-mt-727180d9","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-727180d9","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:19Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"bfd93c96-fdc4-4628-959a-d104daf2e859","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:19Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"},"namespace":"ai-tenant-e2e-mt-727180d9","name":"e2e-lifecycle-model-727180d9","reconcileID":"bfd93c96-fdc4-4628-959a-d104daf2e859","error":"HTTPRoute ai-tenant-e2e-mt-727180d9/e2e-lifecycle-model-727180d9-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-727180d9). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:56:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:56:24Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T22:56:24Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6060ae9d-9302-48c8-901f-f468f57fd564","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:56:24Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6060ae9d-9302-48c8-901f-f468f57fd564","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"cc24a1ca-53d6-44e0-a7e2-bb58b653c6e4","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"c52822c1-c932-416f-9438-33392ec1eb26","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1c6a0de2-1d0a-4dd8-aa94-1deb03bd1fc7","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"17717725-e7fb-4e3a-bb5a-b540794ca660","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-d02f68","reconcileID":"4a2b44fc-c252-4cf1-aa5b-738d596034a9","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-d02f68","reconcileID":"4a2b44fc-c252-4cf1-aa5b-738d596034a9"}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-d02f68","reconcileID":"4a2b44fc-c252-4cf1-aa5b-738d596034a9","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"393ad50c-e6d6-45ce-96bc-d81f12f1b637","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0cae6a5f-3c9f-48d6-b961-7b4328ef7065","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"18199c52-acca-4594-a5db-dd448786678a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-d02f68","reconcileID":"0256c1a5-a5b6-4677-bbeb-39166261c6d1","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-default-int-sub-d02f68","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"75214172-16d3-416d-8ab6-07bf211a96ff","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c3734ef4-2b03-4ab4-a48b-3a204fe594f3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-d02f68","reconcileID":"00989412-61a7-4e5f-9a59-6980202e35be","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-d02f68","reconcileID":"d6e65120-2ef9-4f48-b2de-1f6580bb0022","MaaSAuthPolicy":{"name":"e2e-default-int-auth-d02f68","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"71c6c4cf-40bb-4d08-a3da-48e2548e704f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"45acaba5-faed-4482-b43c-441510707f71","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-d02f68","reconcileID":"c227f5f8-cf2c-4931-9bfa-e98a4fd50ca1","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-default-int-sub-d02f68"}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2f21f4cb-c98e-48a2-bc64-8c904b5c585a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-d02f68","reconcileID":"c227f5f8-cf2c-4931-9bfa-e98a4fd50ca1","MaaSSubscription":{"name":"e2e-default-int-sub-d02f68","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"df1061df-207f-4373-8f13-38b22e7345e6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"fd750637-a3e4-4c92-bff8-db7542fdf096","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:56:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"938c77fd-5b69-4068-b7d1-c974727a5d8f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a5bded77-040d-42cd-aed4-f0b97f97db42","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:53Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-6d12c532/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-policy-6d12c532","reconcileID":"0ff80a3d-2846-4993-b2fd-dfb84179fbb0","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-policy-6d12c532","reconcileID":"0ff80a3d-2846-4993-b2fd-dfb84179fbb0"}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-policy-6d12c532","reconcileID":"0ff80a3d-2846-4993-b2fd-dfb84179fbb0","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8dfaaedd-4e79-42dd-aa2d-15690e564392","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d6b70ac7-cb4e-47de-a89a-f401a70b4598","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9f7de354-cec5-4b4a-a1a8-1b23da1408d5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-10T22:56:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-sub-6d12c532","reconcileID":"6e53d2cb-8723-4797-8067-362e67bc5a3d","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-6d12c532\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"95ac6f59-75cc-48af-a98c-9264d0e0ca26","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-sub-6d12c532","reconcileID":"8f40a519-3291-4933-9a38-5d85299ae841","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-6d12c532/e2e-shared-int-sub-6d12c532","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"456786d6-e80b-4265-8b44-3bd6e07f950c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5dac5a6f-632b-471a-969f-f73fded8bfc7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-10T22:56:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-sub-6d12c532","reconcileID":"510ac536-970b-4d26-a4b0-c2e6a15536e2","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"534b5ef8-0b5d-439b-9804-72e9b4b2a718","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-a7b1512a/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9bee539b-c302-4ac7-91a2-18b92d78903e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-policy-6d12c532","reconcileID":"835f146c-c734-4222-8eca-46472f9dc7cf","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-policy-6d12c532","reconcileID":"835f146c-c734-4222-8eca-46472f9dc7cf"}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-policy-6d12c532","reconcileID":"835f146c-c734-4222-8eca-46472f9dc7cf","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e71b2420-844a-464b-bbe3-4f7e27daf5a3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"63f3ec76-6de0-481b-910c-d5581ee17518","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"832a62d4-7d74-4e7e-89d2-42003615f5d5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-10T22:56:54Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-sub-6d12c532","reconcileID":"7f3267e3-ccb3-4411-aa34-9ad5fbcc670c","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-6d12c532\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e6a547c1-8614-44da-aa84-acc67364aa06","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-sub-6d12c532","reconcileID":"7d675eac-402b-4843-8309-8095cb88962a","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-6d12c532/e2e-shared-int-sub-6d12c532","ai-tenant-e2e-mt-a7b1512a/e2e-shared-int-sub-6d12c532","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e6c65a15-ab40-4f87-a93e-a4c8ee6a3b80","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-sub-6d12c532","reconcileID":"ca92d17c-25ab-4ab3-ba75-3ca78c41643b","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"b28d436b-5bda-4a0c-a83f-d1e0af28962e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-sub-6d12c532","reconcileID":"cfe21cbb-d9fe-4f02-9bdb-3613bd6ecb2c","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"22850d3f-317b-4be8-b0bf-d64ea434a012","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-policy-6d12c532","reconcileID":"0493dbda-c423-43d7-b94c-a141b6a55fec","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"acbc2b4a-3646-4d1f-8579-6c4e3b67e024","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fb9db42c-4742-4b18-be9c-e9b7b32b2bb2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:55Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-sub-6d12c532","reconcileID":"85c941a1-1e20-410a-8931-5a78fb18d698","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-6d12c532"}
{"level":"info","ts":"2026-07-10T22:56:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7616a1c0-7a79-4027-bf2d-5e135c9c655f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:55Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"namespace":"ai-tenant-e2e-mt-6d12c532","name":"e2e-shared-int-sub-6d12c532","reconcileID":"85c941a1-1e20-410a-8931-5a78fb18d698","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-6d12c532"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-a7b1512a/e2e-shared-int-sub-6d12c532","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-10T22:56:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"beff86b2-6b91-4104-bff3-f7fbde56261c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:56:55Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"647e295c-6a5a-4aa2-b566-5b81f538bd50","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-10T22:56:55Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-sub-6d12c532","reconcileID":"285c6f8b-52fc-4e2b-bb6a-4254b737f85c","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-policy-6d12c532","reconcileID":"208740b1-e523-4267-8b36-5ad59909c01c","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0b776f65-455f-4866-9842-8ffd23bfe0a6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5ebf190b-9397-4ed4-9760-c3064cb364fc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-sub-6d12c532","reconcileID":"848de2e0-b506-48fd-90e0-03faf32373dd","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-6d12c532"}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a22b7be6-4145-4ee9-b318-e245657eb3dc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"namespace":"ai-tenant-e2e-mt-a7b1512a","name":"e2e-shared-int-sub-6d12c532","reconcileID":"848de2e0-b506-48fd-90e0-03faf32373dd","MaaSSubscription":{"name":"e2e-shared-int-sub-6d12c532","namespace":"ai-tenant-e2e-mt-a7b1512a"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"75a5503c-fc5a-4660-b25f-a8a3e2727e63","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:57:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"3f2ba13d-6c52-4f29-9258-a99f2e063860","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"92456ab9-5c57-4133-9b3e-4e90ad526f3b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:06Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"891d0bfb-c0aa-4f59-aea4-30f82b8d2772","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:09Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T22:57:09Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"18abd7c4-b45c-4468-95f5-3d8fb21d0685","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:09Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"18abd7c4-b45c-4468-95f5-3d8fb21d0685","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:13Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"6847f809-62d7-496f-a094-730a0afd5a6f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:13Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4183fdb2-baa6-439f-bb28-d4164c89717c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:18Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e98c64c2-8479-47d6-9255-bb182baa6762","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:57:18Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-9f4af73a/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-10T22:57:18Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"namespace":"ai-tenant-e2e-mt-9f4af73a","name":"e2e-policy-9f4af73a","reconcileID":"ecf9979c-a2d8-4e4e-a141-42e15787e969","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:57:18Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"namespace":"ai-tenant-e2e-mt-9f4af73a","name":"e2e-policy-9f4af73a","reconcileID":"ecf9979c-a2d8-4e4e-a141-42e15787e969"}
{"level":"info","ts":"2026-07-10T22:57:18Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"namespace":"ai-tenant-e2e-mt-9f4af73a","name":"e2e-policy-9f4af73a","reconcileID":"ecf9979c-a2d8-4e4e-a141-42e15787e969","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:57:18Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dfb7c16d-a0a6-45a4-b799-53a34f1ce0a2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:57:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:57:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:57:39Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"namespace":"ai-tenant-e2e-mt-9f4af73a","name":"e2e-policy-9f4af73a","reconcileID":"a90cd204-eb93-4bc0-8032-ec93b029bf81","MaaSAuthPolicy":{"name":"e2e-policy-9f4af73a","namespace":"ai-tenant-e2e-mt-9f4af73a"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-10T22:57:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:57:49Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"397fa0d8-d001-46c1-84eb-f3b6704deb75","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:49Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"397fa0d8-d001-46c1-84eb-f3b6704deb75","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:57:49Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"1720b543-8e0b-462b-9946-8fbc0134d13e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:49Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"1720b543-8e0b-462b-9946-8fbc0134d13e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:57:49Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"1720b543-8e0b-462b-9946-8fbc0134d13e","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","tenantID":"e2e-inf-a-c8c45c","subscriptionNamespace":"ai-tenant-e2e-inf-a-c8c45c","gatewayName":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:57:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-c8c45c","deploymentName":"maas-api-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:57:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-c8c45c","deploymentName":"maas-api-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:57:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-c8c45c","deploymentName":"maas-api-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:57:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:57:50Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-c8c45c","deploymentName":"maas-api-e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:57:50Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"1720b543-8e0b-462b-9946-8fbc0134d13e","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:57:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"13adf653-fff6-470f-a5d5-8ba35823b16b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"13adf653-fff6-470f-a5d5-8ba35823b16b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:57:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"23977ec8-e9a4-4bab-b6d2-5e084e43fa11","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"23977ec8-e9a4-4bab-b6d2-5e084e43fa11","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:57:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:57:54Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T22:57:54Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f7752221-71d8-45f8-8971-49dd0a47499a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:54Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f7752221-71d8-45f8-8971-49dd0a47499a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:57Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:57:57Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"9e5b8782-5843-42f7-a1f3-cef5d1f9061d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:57Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"9e5b8782-5843-42f7-a1f3-cef5d1f9061d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:57Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:57:57Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"fdc3afee-a5ae-4cf9-a845-ed9a891128e3","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:57Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"fdc3afee-a5ae-4cf9-a845-ed9a891128e3","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:57Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:57:57Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"fdc3afee-a5ae-4cf9-a845-ed9a891128e3","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","tenantID":"e2e-inf-b-c49b05","subscriptionNamespace":"ai-tenant-e2e-inf-b-c49b05","gatewayName":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:57:57Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-c49b05","deploymentName":"maas-api-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:57:57Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-c49b05","deploymentName":"maas-api-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:57:57Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-c49b05","deploymentName":"maas-api-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:57:57Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"02cd03e2-bf66-4c15-9a42-2e551fa5fe08","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:57Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"bf35f9db-6188-47ba-823e-3f548a82be2b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:57:58Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:57:58Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-c49b05","deploymentName":"maas-api-e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:57:58Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"fdc3afee-a5ae-4cf9-a845-ed9a891128e3","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:58Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:57:58Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"08a836b3-fda1-4002-9802-08d8d6604555","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:58Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"08a836b3-fda1-4002-9802-08d8d6604555","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:57:58Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:57:58Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"8911a337-ed0b-4ee9-825e-11ab43a4d971","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:57:58Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"8911a337-ed0b-4ee9-825e-11ab43a4d971","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"c7d4371b-21c8-41dd-8bf5-db2b0e98511f","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"6977e207-4f9d-4389-96f6-dea719fd67c5","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"8d6a01c8-d06e-464b-a5fe-064a6a2ae4a6","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"error","ts":"2026-07-10T22:58:12Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"dfe1f572-e7d2-4564-9a52-91a2dfaa6556","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"test-model-c8c45c-sub\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"7b838864-088c-4011-b7ad-3bf139be8330","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"a62f6254-3ebd-44ff-aa8c-ffc54bb8e235","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"test-model-c8c45c","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c-sub"]}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"48e06509-6328-4dfc-8a0d-629b8850c943","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"5c5abb48-e49d-47dc-9d74-da930d3e825b","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"9c4e8e7c-4c70-477a-a2d2-989464eee953","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"077e3f8f-17e7-49cc-b8b5-15e7355f38e5","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"dadd9089-9a8f-4b7a-ae9d-be12208e1bf1","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-c8c45c","tenantID":"e2e-inf-a-c8c45c","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-10T22:58:12Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"077e3f8f-17e7-49cc-b8b5-15e7355f38e5","name":"test-model-c8c45c","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"test-model-c8c45c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"b432f005-7732-46b6-af73-f982a8e24565","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"dadd9089-9a8f-4b7a-ae9d-be12208e1bf1"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"dadd9089-9a8f-4b7a-ae9d-be12208e1bf1","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"e2e-inf-a-c8c45c-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"73c6a02d-6bb2-4ccd-8d46-a52bc9999844","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:12Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"a416112d-8303-4c78-a63a-2565123bb046","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-c8c45c","tenantID":"e2e-inf-a-c8c45c","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:58:12Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"a416112d-8303-4c78-a63a-2565123bb046"}
{"level":"info","ts":"2026-07-10T22:58:13Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"a416112d-8303-4c78-a63a-2565123bb046","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"e2e-inf-a-c8c45c-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:58:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"43e24e6c-3bcf-49c7-84b3-d6f159dcefd8","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:13Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:58:13Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"cdcbbdc5-f97c-4156-9bb6-a3f66154417e","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-c8c45c","tenantID":"e2e-inf-a-c8c45c","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:58:13Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"2d7f5ae5-af21-4142-95e2-85843d74dd39","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:13Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"cdcbbdc5-f97c-4156-9bb6-a3f66154417e"}
{"level":"info","ts":"2026-07-10T22:58:13Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"cdcbbdc5-f97c-4156-9bb6-a3f66154417e","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"e2e-inf-a-c8c45c-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"0b3e7e98-e47d-4704-889a-befe7ddeab50","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"928b7be5-53f9-458b-ac09-c65ca677c089","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"a249ac35-4d57-4055-ba1e-57e2f2029a5e","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:21Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"768696eb-1454-44b4-b40b-e4130b57bc99","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-c8c45c","tenantID":"e2e-inf-a-c8c45c","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"4e34d2fa-3015-44f8-9872-6d4de59dce23","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"768696eb-1454-44b4-b40b-e4130b57bc99"}
{"level":"info","ts":"2026-07-10T22:58:21Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"768696eb-1454-44b4-b40b-e4130b57bc99","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"e2e-inf-a-c8c45c-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:58:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:58:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:58:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"f6f57d48-bc99-4584-b5ba-7c1a4959821a","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:58:25Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:58:25Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"18fe79a3-945d-4cba-8586-f2c1c67db079","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-c8c45c","tenantID":"e2e-inf-a-c8c45c","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:58:25Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"78abdea6-d2c4-431c-82f2-d123b1684c14","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:25Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"18fe79a3-945d-4cba-8586-f2c1c67db079"}
{"level":"info","ts":"2026-07-10T22:58:25Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"18fe79a3-945d-4cba-8586-f2c1c67db079","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"e2e-inf-a-c8c45c-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:58:26Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"e632099f-0610-4bcf-9775-e6ce512d3de2","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:27Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"685c3faa-457c-43b2-ab89-1c9dd7f2e45b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:27Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d4c228f7-049d-4999-bf77-6b3611b7d3b6","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:58:35Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:58:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"b0f80f31-f40a-4109-85de-0cbf1232a71a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:58:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"b0f80f31-f40a-4109-85de-0cbf1232a71a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:58:39Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T22:58:39Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"b3c052b2-5e91-416d-ae42-699ebaf5f5d7","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:58:39Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"b3c052b2-5e91-416d-ae42-699ebaf5f5d7","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:58:43Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:58:43Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"f6d8e5bc-dc85-4c97-ad20-70a99569a316","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:58:43Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"f6d8e5bc-dc85-4c97-ad20-70a99569a316","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:58:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:58:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:59:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"570a1304-21c6-41c6-8c9d-ac55f9bcb232","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"3121b7a8-612b-463b-8b14-d145014a0e11","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"b0518a6d-a4ed-4a18-b1cd-01480080eec9","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:04Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"e0c46b04-cb53-47ee-8e46-69879e8d223e","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"test-model-c49b05","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-inf-b-c49b05/test-model-c49b05-sub"]}
{"level":"info","ts":"2026-07-10T22:59:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"e7381375-a3ab-48ac-b9d6-5d76bd804487","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"0162a38f-a832-4251-a2f5-f4f2a1792625","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"7f6c6e43-7253-4595-8fa4-5acc541d7799","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:05Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"09055d2d-f3bd-4d20-9af6-e325aeb67a03","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"d841979f-c21f-4a47-b0a8-34a5a7ce4a85","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-c49b05","tenantID":"e2e-inf-b-c49b05","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"10ba7795-6966-49e7-a952-dda454e2e78a","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"d841979f-c21f-4a47-b0a8-34a5a7ce4a85"}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"d841979f-c21f-4a47-b0a8-34a5a7ce4a85","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"e2e-inf-b-c49b05-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"8827108d-7f53-4783-ae00-d5f60fc04347","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:05Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"643690e1-2bf9-4173-9843-21e1f7233bd7","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-c49b05","tenantID":"e2e-inf-b-c49b05","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"643690e1-2bf9-4173-9843-21e1f7233bd7"}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"643690e1-2bf9-4173-9843-21e1f7233bd7","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"e2e-inf-b-c49b05-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:59:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"c9c5628a-1c4a-4d03-ac9b-659ed19cf9e5","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"0bc8be69-32e4-410f-a260-49ed842d1965","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"3c8bc97f-6943-4c1f-832f-13b15e740ba0","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"686ca11a-dd49-468f-87fd-15734f497974","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"7d884282-4978-4613-8d75-4a139b852d53","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"884ae468-6cf1-4acb-8fbd-9314baac9332","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"a439d435-bdf1-4526-9d47-1a1c1bcc6320","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:10Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"219d57df-f3aa-4ed9-91c3-fcd39f768ff8","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-c49b05","tenantID":"e2e-inf-b-c49b05","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"c35c53cd-73de-4755-81b9-8c16c60da924","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"219d57df-f3aa-4ed9-91c3-fcd39f768ff8"}
{"level":"info","ts":"2026-07-10T22:59:10Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"219d57df-f3aa-4ed9-91c3-fcd39f768ff8","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"e2e-inf-b-c49b05-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:59:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:59:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"e20dd8fb-a49a-4a7d-96aa-488f17370ec6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:59:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"e20dd8fb-a49a-4a7d-96aa-488f17370ec6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:59:20Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"29c510e5-d9fa-47d7-b425-a1f1f6867d60","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:21Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"fcf703f9-8150-4920-8403-60cf0a429afd","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:21Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"fd2125ce-ff64-4f3e-8f70-83b317105831","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:22Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"56968861-5d6d-4bb6-8d12-3dd80957e843","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:59:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:59:24Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T22:59:24Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"a2071d3d-42fa-4ba9-b271-94ab134f566b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:59:24Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"a2071d3d-42fa-4ba9-b271-94ab134f566b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:59:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"3d322163-8400-48ac-acb0-f51ad8d20996","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:28Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:28Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"18ce35f5-152a-41a0-ba4d-b2c98d8b18ca","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-c49b05","tenantID":"e2e-inf-b-c49b05","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-10T22:59:28Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"e573d646-2fed-47df-a316-dee51cae2956","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:28Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T22:59:28Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"84e4b4f5-c8b6-42c9-8414-069835ffecdd","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:59:28Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"84e4b4f5-c8b6-42c9-8414-069835ffecdd","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:59:28Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"18ce35f5-152a-41a0-ba4d-b2c98d8b18ca"}
{"level":"info","ts":"2026-07-10T22:59:28Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"18ce35f5-152a-41a0-ba4d-b2c98d8b18ca","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"e2e-inf-b-c49b05-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-auth","reconcileID":"fcee7f1e-5318-4c35-af84-f9868abf10fa","MaaSAuthPolicy":{"name":"test-model-c8c45c-auth","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"d7898a99-ff82-4a8d-911e-4aac21de3d1e","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"e77f9518-aa89-4c2a-8dfe-9fe00e692b73","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"de2f4bdc-b7b5-4d03-a5de-79fde957a37f","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"668a943b-3ba2-45cb-87c5-f3d4fc273c6c","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"ef427718-2d92-48dd-95b7-126d3cf6cb42","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c","subscription":"test-model-c8c45c-sub"}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"b9233ab3-87af-4b74-a768-109ad6c16a99","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"ef427718-2d92-48dd-95b7-126d3cf6cb42","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c-sub","reconcileID":"ef427718-2d92-48dd-95b7-126d3cf6cb42","MaaSSubscription":{"name":"test-model-c8c45c-sub","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"name":"maas-trlp-test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c","model":"ai-tenant-e2e-inf-a-c8c45c/test-model-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"test-model-c8c45c","reconcileID":"9217e7a2-b975-4a23-ade1-285ad33625a1","MaaSModelRef":{"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"routeName":"test-model-c8c45c-kserve-route","namespace":"ai-tenant-e2e-inf-a-c8c45c","llmisvcName":"test-model-c8c45c","gateway":"openshift-ingress/e2e-inf-a-c8c45c","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"5ff884e1-c4da-4f25-9efc-e3aef060b0f3","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-auth","reconcileID":"05f896b6-aabf-4704-b257-40cc4b9c8664","MaaSAuthPolicy":{"name":"test-model-c49b05-auth","namespace":"ai-tenant-e2e-inf-b-c49b05"},"model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"e48f44b9-e0d2-40e7-8216-5337bb825ab4","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"7bb16b83-5039-4feb-b82d-3445c903b965","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"2da25391-a345-412a-b5c6-ed06e253e25b","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"8eb1d67a-ee49-41e2-b795-ecad694f5590","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"b32e4bda-6f05-46ff-b321-31a9f0d0819e","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05","subscription":"test-model-c49b05-sub"}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"74dcd73e-bb3a-4859-88e3-5d57837f1bbb","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"b32e4bda-6f05-46ff-b321-31a9f0d0819e","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05-sub","reconcileID":"b32e4bda-6f05-46ff-b321-31a9f0d0819e","MaaSSubscription":{"name":"test-model-c49b05-sub","namespace":"ai-tenant-e2e-inf-b-c49b05"},"name":"maas-trlp-test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05","model":"ai-tenant-e2e-inf-b-c49b05/test-model-c49b05"}
{"level":"info","ts":"2026-07-10T22:59:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"test-model-c49b05","reconcileID":"c84981a5-b901-447a-ab73-21de58eed921","MaaSModelRef":{"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"},"routeName":"test-model-c49b05-kserve-route","namespace":"ai-tenant-e2e-inf-b-c49b05","llmisvcName":"test-model-c49b05","gateway":"openshift-ingress/e2e-inf-b-c49b05","hostnames":[]}
{"level":"info","ts":"2026-07-10T22:59:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"error","ts":"2026-07-10T22:59:37Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"96f49dd2-3c8b-4c02-8afb-a63970090cda","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T22:59:37Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"96f49dd2-3c8b-4c02-8afb-a63970090cda","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T22:59:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","namespace":"ai-tenant-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","count":0}
{"level":"info","ts":"2026-07-10T22:59:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-c8c45c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-c8c45c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-c8c45c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-c8c45c"},"tenantIdentifier":"e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","tenant":"e2e-inf-a-c8c45c","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-10T22:59:37Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-c8c45c","deploymentName":"maas-api-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","gvk":"Deployment","name":"maas-api-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","gvk":"Service","name":"maas-api-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-c8c45c"},"namespace":"ai-tenant-e2e-inf-a-c8c45c","name":"default-tenant","reconcileID":"d05060b0-9ec3-4bdd-a290-187935b8b51e","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-a-c8c45c"}
{"level":"info","ts":"2026-07-10T22:59:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"1c9c27bd-ebae-499a-951a-7b2720d2b8ac","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"b9f74e5c-3e06-455b-a291-9fff08fa1ff6","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"2b60da03-df46-4a67-8eda-42566c568874","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"33237884-b86b-4ef3-bcad-293d667cd341","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T22:59:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T22:59:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:00:09Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T23:00:09Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c4c0e9f4-d421-438a-8c94-45aa3e8f296e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:09Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c4c0e9f4-d421-438a-8c94-45aa3e8f296e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:00:13Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T23:00:13Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"664174cf-37bf-4cce-9cf7-be821c055785","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:13Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"664174cf-37bf-4cce-9cf7-be821c055785","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:00:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"error","ts":"2026-07-10T23:00:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"751087ca-0a47-4a30-97b4-634f70e6b4a7","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"751087ca-0a47-4a30-97b4-634f70e6b4a7","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:00:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","namespace":"ai-tenant-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","count":0}
{"level":"info","ts":"2026-07-10T23:00:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-c49b05/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-c49b05","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-c49b05","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-c49b05"},"tenantIdentifier":"e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","tenant":"e2e-inf-b-c49b05","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-10T23:00:20Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-c49b05","deploymentName":"maas-api-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","gvk":"Deployment","name":"maas-api-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","gvk":"Service","name":"maas-api-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-c49b05"},"namespace":"ai-tenant-e2e-inf-b-c49b05","name":"default-tenant","reconcileID":"6bb494c0-7142-4793-a810-b29890e0c8ed","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-b-c49b05"}
{"level":"info","ts":"2026-07-10T23:00:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:00:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:00:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"f9cbc158-ce0f-4cc1-8209-f7b3ccaef25d","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T23:00:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"2b5ff06c-bf8b-43fd-869b-538cb057fbf7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T23:00:31Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:00:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"42710d52-9746-4041-b5b1-ff82098b0537","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"42710d52-9746-4041-b5b1-ff82098b0537","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:00:31Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:00:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"e3146a8b-96ae-4dbe-9f2a-21c71da34a30","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"e3146a8b-96ae-4dbe-9f2a-21c71da34a30","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:00:31Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:00:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"aa699a70-96a0-4c6e-82c4-b280ad488173","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"aa699a70-96a0-4c6e-82c4-b280ad488173","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:00:32Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"87b63ee1-592a-4246-9bd3-bf7a73bb9986","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T23:00:32Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d0c44d7c-19b0-491c-ae2c-44d6e2659d2c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-10T23:00:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:00:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:00:54Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T23:00:54Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"018c9343-599a-406d-b935-dddb3ce28438","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:00:54Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"018c9343-599a-406d-b935-dddb3ce28438","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:01:16Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:01:16Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"378055d7-7ee9-4d21-9f23-17308ceef4b9","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:01:16Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"378055d7-7ee9-4d21-9f23-17308ceef4b9","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:01:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:01:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:01:39Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T23:01:39Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"1cdd8922-4d12-4997-9d51-d5b904fb3446","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:01:39Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"1cdd8922-4d12-4997-9d51-d5b904fb3446","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:01:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:01:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:02:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:02:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"1178b2b3-7bfe-4204-893b-aadd9187f591","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:02:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"1178b2b3-7bfe-4204-893b-aadd9187f591","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:02:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:02:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:02:24Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T23:02:24Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"3aa9ae83-3e4f-45f1-be15-c3082e445e03","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:02:24Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"3aa9ae83-3e4f-45f1-be15-c3082e445e03","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:02:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:02:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"6184fbba-deea-41ba-a48e-2fddb899f259","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:02:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"6184fbba-deea-41ba-a48e-2fddb899f259","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:02:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:02:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:03:09Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-10T23:03:09Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"2d49bac4-804c-4c65-b6d0-e2da116a0a68","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:03:09Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"2d49bac4-804c-4c65-b6d0-e2da116a0a68","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:03:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:03:22Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:03:31Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:03:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"f8c5c318-8817-4a40-936f-f71d002f71fa","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:03:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"f8c5c318-8817-4a40-936f-f71d002f71fa","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:03:40Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"error","ts":"2026-07-10T23:03:40Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"0ba63e62-37db-4493-ac8b-b02fdf8fc555","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-10T23:03:40Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"0ba63e62-37db-4493-ac8b-b02fdf8fc555","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-10T23:03:40Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","namespace":"ai-tenant-e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","count":0}
{"level":"info","ts":"2026-07-10T23:03:40Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-54ec41/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-54ec41","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-54ec41","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-54ec41"},"tenantIdentifier":"e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","tenant":"e2e-api-a-54ec41","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-10T23:03:40Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-api-a-54ec41","deploymentName":"maas-api-e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","gvk":"Deployment","name":"maas-api-e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","gvk":"Service","name":"maas-api-e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","gvk":"HTTPRoute","name":"maas-api-route-e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:40Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-54ec41"},"namespace":"ai-tenant-e2e-api-a-54ec41","name":"default-tenant","reconcileID":"cedd2cd0-38c1-433d-8fa2-0fda951ba736","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-api-a-54ec41"}
{"level":"info","ts":"2026-07-10T23:03:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-10T23:03:52Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}