{"level":"error","ts":"2026-07-13T14:56:12Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"91b15718-fe06-4074-a148-ac365cd68f9f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:12Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"91b15718-fe06-4074-a148-ac365cd68f9f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:56:12Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"error","ts":"2026-07-13T14:56:12Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"8dab4e8a-04a7-4fe2-80bd-c6284324b10d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:12Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"8dab4e8a-04a7-4fe2-80bd-c6284324b10d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:56:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:56:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:56:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"47ecdf39-f271-4f8f-aab4-2c39d8c50ea6","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"18baf6a4-49fc-4cbb-ab0c-26805410c7d7","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"f2a9eae8-2c9d-407d-bc4d-54a4a6a2e2d4","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:25Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"86f41c6c-f287-4c76-ab4e-b1aca4bf8c9c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"a657cb85-a5bb-4a00-ab8f-16a5313b3a89","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:30Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"9cc6bb24-1594-46db-9816-1f7d4e70ab0a","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"83094f8f-39dd-4986-8a89-b01519136a68","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"77078440-c0de-4156-bb0a-e3873b175069","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"00c8d4f0-eac4-4b83-bfdf-d7b7c4a90f7e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"2d292c44-c9c1-4426-bea4-d3e21c4304fa","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T14:56:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0ea348b1-d6cf-4bed-afba-f89e4a0730da","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0ea348b1-d6cf-4bed-afba-f89e4a0730da","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:56:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"c1bf930e-9ffb-4591-ae3a-89e3f0903846","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"00787eeb-2f06-4eb9-a8dd-df396ed95f78","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"50e19688-98b8-4f35-929f-88b56643480b","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:42Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"efd39baa-6f1b-4fd6-9ba5-defe07a8c1b3","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-575864a3","tenantID":"e2e-mt-575864a3","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"efd39baa-6f1b-4fd6-9ba5-defe07a8c1b3"}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"efd39baa-6f1b-4fd6-9ba5-defe07a8c1b3","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"e2e-mt-575864a3-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"081b40af-d919-4bfe-a2f6-4a0d4b073457","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"0ba137fa-ac70-4011-8a08-8648f286f984","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"fdfb280d-7095-4695-9937-978ec4531895","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:43Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"error","ts":"2026-07-13T14:56:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"7d83aacc-a676-44c1-ac75-67e6b9752aec","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-sub-575864a3\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"d1cea94a-04ab-4328-b9b0-6247257ec848","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-575864a3","tenantID":"e2e-mt-575864a3","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"d168b2cd-27a2-4076-bf67-7459e31b9555","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"d1cea94a-04ab-4328-b9b0-6247257ec848"}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"d1cea94a-04ab-4328-b9b0-6247257ec848","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"e2e-mt-575864a3-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"2512e933-c6b3-4f32-813c-0151443a8bb6","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","model":"e2e-lifecycle-model-575864a3","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-mt-575864a3/e2e-sub-575864a3"]}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"a5573300-a920-477d-be1b-5262d9772ae7","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"93177a94-2a77-4a13-a5f1-fa2ff7f987af","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"82773c8d-22d2-4f06-b625-59a64ce43927","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a9fcb93c-634c-48f7-8442-97fcbf8db2a1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"c0a5fd99-f257-4ff2-b604-d67725a24842","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"ee718093-b89c-4faa-8513-527bb83be4b8","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"d746a252-0c33-4d93-8fd6-c99baa8c47b8","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"6c914df9-a354-4567-8d64-0ddc3332de2c","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-575864a3","tenantID":"e2e-mt-575864a3","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"11476353-5df2-4d37-8cef-13b5563b28c0","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"6c914df9-a354-4567-8d64-0ddc3332de2c"}
{"level":"info","ts":"2026-07-13T14:56:47Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"6c914df9-a354-4567-8d64-0ddc3332de2c","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"e2e-mt-575864a3-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"591176e6-5eb8-4315-9497-cc44714638e2","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-575864a3","tenantID":"e2e-mt-575864a3","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-13T14:56:48Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"fc7b4dd5-fcba-46a8-9a3a-68b19ee7e908","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:48Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"fc7b4dd5-fcba-46a8-9a3a-68b19ee7e908","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"8d586c0f-a260-4562-9d9b-dbc6493ab192","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","namespace":"ai-tenant-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Deleting MaaSAuthPolicy","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","count":1}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-575864a3/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-575864a3","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-575864a3","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-575864a3"},"tenantIdentifier":"e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","tenant":"e2e-mt-575864a3","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T14:56:48Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-575864a3","deploymentName":"maas-api-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"d862cfd9-d4c8-47a2-b8c1-e043dbc7be11","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","namespace":"ai-tenant-e2e-mt-575864a3","llmisvcName":"e2e-lifecycle-model-575864a3","gateway":"openshift-ingress/e2e-mt-575864a3","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"591176e6-5eb8-4315-9497-cc44714638e2"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"591176e6-5eb8-4315-9497-cc44714638e2","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"e2e-mt-575864a3-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","gvk":"Deployment","name":"maas-api-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"a6184b3d-0697-4896-bd04-6c974a44b239","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","gvk":"Service","name":"maas-api-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"default-tenant","reconcileID":"e71eff82-5f57-42da-8669-0b24d54428a6","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:48Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-policy-575864a3","reconcileID":"079e077b-7ca0-406b-89d9-9e03388a2e92","MaaSAuthPolicy":{"name":"e2e-policy-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:49Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"ddb5270a-db85-4917-bd53-d499d5f06e70","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3","subscription":"e2e-sub-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:49Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"ddb5270a-db85-4917-bd53-d499d5f06e70","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3"}
{"level":"info","ts":"2026-07-13T14:56:49Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-sub-575864a3","reconcileID":"ddb5270a-db85-4917-bd53-d499d5f06e70","MaaSSubscription":{"name":"e2e-sub-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"name":"maas-trlp-e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3","model":"ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"9589693c-bf96-4583-8192-cb6d13d52398","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"9589693c-bf96-4583-8192-cb6d13d52398","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"9589693c-bf96-4583-8192-cb6d13d52398","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"838457d3-b7f4-4519-b17b-bebc092b05e4","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"838457d3-b7f4-4519-b17b-bebc092b05e4","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"838457d3-b7f4-4519-b17b-bebc092b05e4","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"62a37e25-eacc-4b3b-a334-0ef7629f02be","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"62a37e25-eacc-4b3b-a334-0ef7629f02be","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"62a37e25-eacc-4b3b-a334-0ef7629f02be","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"2b265c6b-5b15-4f6e-9e6e-d268314598fb","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"2b265c6b-5b15-4f6e-9e6e-d268314598fb","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"2b265c6b-5b15-4f6e-9e6e-d268314598fb","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"c28537e1-2f29-4cf5-813c-1a042f3b20f3","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"c28537e1-2f29-4cf5-813c-1a042f3b20f3","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"c28537e1-2f29-4cf5-813c-1a042f3b20f3","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"a4ea0f70-c4f2-4439-afaf-bf0a9a19360c","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"a4ea0f70-c4f2-4439-afaf-bf0a9a19360c","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"a4ea0f70-c4f2-4439-afaf-bf0a9a19360c","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"f50678e8-1a6d-49a2-82af-33eff2cd5463","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"f50678e8-1a6d-49a2-82af-33eff2cd5463","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"f50678e8-1a6d-49a2-82af-33eff2cd5463","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"e54ac3a1-5ecb-4d32-9489-108fe0664980","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"e54ac3a1-5ecb-4d32-9489-108fe0664980","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"e54ac3a1-5ecb-4d32-9489-108fe0664980","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:55Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"345b78c8-38d8-45d9-8b8a-c1b0b6f3bdc5","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:55Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"345b78c8-38d8-45d9-8b8a-c1b0b6f3bdc5","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:55Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"345b78c8-38d8-45d9-8b8a-c1b0b6f3bdc5","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:55Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"95679968-11ea-4618-ab0c-62488b84c8f9","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:55Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"95679968-11ea-4618-ab0c-62488b84c8f9","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:55Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"95679968-11ea-4618-ab0c-62488b84c8f9","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:58Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"2cd09403-5bee-4bdb-8956-1c36343bc5fa","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:58Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"2cd09403-5bee-4bdb-8956-1c36343bc5fa","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:56:58Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"2cd09403-5bee-4bdb-8956-1c36343bc5fa","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:03Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"d15ba12d-7f0a-43cd-a4ea-3e26d7a9059e","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:03Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"d15ba12d-7f0a-43cd-a4ea-3e26d7a9059e","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:03Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"d15ba12d-7f0a-43cd-a4ea-3e26d7a9059e","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:57:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a701fe22-a119-4994-8ee1-6bf3a60d89c7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0f772c9a-c336-4007-a438-b7b6ac022aa1","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-13T14:57:13Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"70d6aeb4-a8d3-47e9-a6ad-a7642da63045","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:13Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"70d6aeb4-a8d3-47e9-a6ad-a7642da63045","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:13Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"70d6aeb4-a8d3-47e9-a6ad-a7642da63045","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:57:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:57:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:57:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T14:57:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c27313de-1289-4fe9-b7b8-679851860e9f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c27313de-1289-4fe9-b7b8-679851860e9f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:33Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"1ee8e4c5-1d43-4877-81fa-6f57afbb7c90","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:33Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"1ee8e4c5-1d43-4877-81fa-6f57afbb7c90","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:33Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"1ee8e4c5-1d43-4877-81fa-6f57afbb7c90","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:37Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"015446ac-c2a8-46ea-a8b2-b60e2f29cdbc","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"routeName":"e2e-lifecycle-model-575864a3-kserve-route","routeNamespace":"ai-tenant-e2e-mt-575864a3","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-575864a3","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:37Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"015446ac-c2a8-46ea-a8b2-b60e2f29cdbc","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:57:37Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-575864a3","namespace":"ai-tenant-e2e-mt-575864a3"},"namespace":"ai-tenant-e2e-mt-575864a3","name":"e2e-lifecycle-model-575864a3","reconcileID":"015446ac-c2a8-46ea-a8b2-b60e2f29cdbc","error":"HTTPRoute ai-tenant-e2e-mt-575864a3/e2e-lifecycle-model-575864a3-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-575864a3). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:57:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"09e1a50d-4ac8-43bc-944f-768869f76a88","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"99a9bfdf-a05f-4822-8d09-d50860c1f675","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fd04f5ab-d3b7-4fa5-aa15-1b4b10d41780","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"65256487-41ac-43af-a654-b27c75f73e46","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:45Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-17af7b","reconcileID":"6aa302bd-5cc1-4a23-96ce-d65711497254","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-17af7b","reconcileID":"6aa302bd-5cc1-4a23-96ce-d65711497254"}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-17af7b","reconcileID":"6aa302bd-5cc1-4a23-96ce-d65711497254","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"68203c51-9593-4f67-b53f-ff4d7fe8d4fd","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1246db0b-7dc5-4592-a410-84197b618e2e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4259a483-9cb3-4210-8bee-ce3d89de9000","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T14:57:45Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-17af7b","reconcileID":"46758a14-af8e-49fd-89cb-f084ac1ca2fc","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-default-int-sub-17af7b\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"552976aa-a100-484d-8d35-085c00fa0888","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-17af7b","reconcileID":"d3345407-925f-4478-b834-ed747fd30aef","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-default-int-sub-17af7b","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bfc9d89f-9d16-4a26-b6c7-fa13c8b21834","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"104e395b-3c96-43c7-aa50-3e97bd08eec0","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:45Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-17af7b","reconcileID":"beb5cea5-aa56-4394-bd0c-5e9d509ff312","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-17af7b","reconcileID":"b301f1ec-5e5d-4ab6-9dfb-7bd100bdf358","MaaSAuthPolicy":{"name":"e2e-default-int-auth-17af7b","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"159b1a0b-53e3-41f9-ab16-9e9b7b95fb2d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7ca63f43-a3b2-4424-9858-9127949755d2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-17af7b","reconcileID":"937d7b42-e018-4c8b-beb8-11f578d984d1","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-default-int-sub-17af7b"}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"40a1fbde-1b5f-4c13-9b1b-2dad2dbd5875","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-17af7b","reconcileID":"937d7b42-e018-4c8b-beb8-11f578d984d1","MaaSSubscription":{"name":"e2e-default-int-sub-17af7b","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"aa48fef6-9e43-4105-a5fe-d8df35e60d8d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"85eb7631-5f34-4d13-864d-4fd2074fc18f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"35ada5ce-2958-465f-b688-9f080d48a890","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fddcc39e-4334-4246-a5ad-17581e1d17e7","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"80795273-0831-4d9d-bff5-d0841c33e2a2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:47Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-090282e7/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-policy-090282e7","reconcileID":"215bf954-6a51-4ade-893e-292228061de4","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-policy-090282e7","reconcileID":"215bf954-6a51-4ade-893e-292228061de4"}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-policy-090282e7","reconcileID":"215bf954-6a51-4ade-893e-292228061de4","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a685edda-50f0-4d50-a3e9-cedd968b463b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"869a856a-2b96-4cd9-b8bb-195adddff925","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f02cbe70-a078-4933-8003-d0c7ba71f842","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-sub-090282e7","reconcileID":"04eaf26b-33c8-4907-8608-17002ff342f8","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-090282e7/e2e-shared-int-sub-090282e7","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b4e8ae77-501d-44ca-8aa1-da01ea562f2f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"e0a4a54f-248f-4722-bb18-4f8e291bbff3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-sub-090282e7","reconcileID":"7c4f394f-3378-4f13-b557-32352b01aabd","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a112ab41-9565-43d5-acee-d9c96ab91730","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c31c97db-0cce-40de-889b-250947ba45b9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-5f00a279/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-policy-090282e7","reconcileID":"c254cff9-a38f-4696-ac3d-ce3d0650e71f","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-policy-090282e7","reconcileID":"c254cff9-a38f-4696-ac3d-ce3d0650e71f"}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-policy-090282e7","reconcileID":"c254cff9-a38f-4696-ac3d-ce3d0650e71f","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3b1c72f6-52d6-4804-9fda-1a8e8151f00b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ac984f8a-1916-43d7-9df5-d7ef8fb99b32","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f0b5172a-7787-427a-adfd-12bb197f0ccb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"32eaa5bf-29d0-4315-a45b-db8049d8004f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"86704041-98ae-45b5-ab3b-7af89749e2b5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-sub-090282e7","reconcileID":"8312b253-0a84-4df4-97ba-c7fa529dc07f","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-090282e7/e2e-shared-int-sub-090282e7","ai-tenant-e2e-mt-5f00a279/e2e-shared-int-sub-090282e7","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c24539ea-a38b-4d8c-a85d-5b64a98f1433","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"e085e27e-f9e1-4711-892c-4ae56f3668d6","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-sub-090282e7","reconcileID":"88ae2da1-94a5-44a2-9c8a-b4ccac4e7190","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-sub-090282e7","reconcileID":"1639e614-a40f-4b56-9ea4-af0e42b80456","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T14:57:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:57:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-policy-090282e7","reconcileID":"0dcfcec1-f364-4a28-a9b7-7775c69bbe08","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"70b9d553-a470-46f4-bfde-73f93d759c70","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"329a191c-bf8d-4f5f-b42d-c5271e16830b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:49Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-sub-090282e7","reconcileID":"65e07c59-1eaa-4d15-8428-54fc05dae8ef","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-090282e7"}
{"level":"info","ts":"2026-07-13T14:57:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fefe6587-e38a-4077-8946-606d398864c2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:49Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"namespace":"ai-tenant-e2e-mt-090282e7","name":"e2e-shared-int-sub-090282e7","reconcileID":"65e07c59-1eaa-4d15-8428-54fc05dae8ef","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-090282e7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-5f00a279/e2e-shared-int-sub-090282e7","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T14:57:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e2575fc5-2d54-4261-92e1-890850e6b7e4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-sub-090282e7","reconcileID":"312b6e34-fbcd-4da8-aaa0-f6f096bfb374","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"8bf848b3-c773-42c5-8b22-a801e56aceb3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"afbf6473-7da2-484d-b940-3c222b00e850","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-sub-090282e7","reconcileID":"0f401e2f-0293-4583-bb45-755da2a7b2bb","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T14:57:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"4e948ad5-ece8-483a-817e-a56728d2ab09","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:55Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-policy-090282e7","reconcileID":"250eb142-8d77-492c-a67f-5c3beee8d981","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T14:57:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d396bab9-2573-4f63-8e3b-d1ff6789bf37","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0a8de146-5151-4126-8de8-d6d2e011fb27","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:55Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-sub-090282e7","reconcileID":"c1440b2f-763d-4cef-9088-758dea9a85d2","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-090282e7"}
{"level":"info","ts":"2026-07-13T14:57:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2030f894-732f-49aa-b91f-ba2f610f1842","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:55Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"namespace":"ai-tenant-e2e-mt-5f00a279","name":"e2e-shared-int-sub-090282e7","reconcileID":"c1440b2f-763d-4cef-9088-758dea9a85d2","MaaSSubscription":{"name":"e2e-shared-int-sub-090282e7","namespace":"ai-tenant-e2e-mt-5f00a279"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T14:57:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3525622a-0cd9-4ad4-b8d7-bc9e0df68ef1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:57:56Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c64f91e3-f6bc-466d-9785-9a2be9e53bfa","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:57:59Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"05cdcf05-9a8f-4cfa-86a2-7932738f3500","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:58:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T14:58:05Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"46acc3b0-45e8-427c-bef5-b1d4f6ebde61","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:05Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"46acc3b0-45e8-427c-bef5-b1d4f6ebde61","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:07Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ffe97f7b-b687-44bb-8e2b-0d144a9cb8c4","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:58:08Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"220ed7e7-b4d5-4b22-a691-604c4fb3b4b7","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:58:13Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-ddfd38ad/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T14:58:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8ba0e69f-75a7-47f1-a4d6-fcab8d0482b4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:58:13Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"namespace":"ai-tenant-e2e-mt-ddfd38ad","name":"e2e-policy-ddfd38ad","reconcileID":"fc940f22-46db-47ee-9f40-745b36dcbe76","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:58:13Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"namespace":"ai-tenant-e2e-mt-ddfd38ad","name":"e2e-policy-ddfd38ad","reconcileID":"fc940f22-46db-47ee-9f40-745b36dcbe76"}
{"level":"info","ts":"2026-07-13T14:58:13Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"namespace":"ai-tenant-e2e-mt-ddfd38ad","name":"e2e-policy-ddfd38ad","reconcileID":"fc940f22-46db-47ee-9f40-745b36dcbe76","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:58:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"befcc03c-e34e-4c46-9c0d-f65b1aaa3e2b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:58:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:58:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:58:34Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"namespace":"ai-tenant-e2e-mt-ddfd38ad","name":"e2e-policy-ddfd38ad","reconcileID":"c0783601-312a-49ca-8c6d-9b16f9d9eb02","MaaSAuthPolicy":{"name":"e2e-policy-ddfd38ad","namespace":"ai-tenant-e2e-mt-ddfd38ad"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T14:58:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T14:58:44Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"3d29c7c8-6542-4c27-a329-8428227b451b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:44Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"3d29c7c8-6542-4c27-a329-8428227b451b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T14:58:44Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"ecda381a-9b3e-44c1-87ee-228fc955146a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:44Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"ecda381a-9b3e-44c1-87ee-228fc955146a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:58:44Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"ecda381a-9b3e-44c1-87ee-228fc955146a","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","tenantID":"e2e-inf-a-d6591f","subscriptionNamespace":"ai-tenant-e2e-inf-a-d6591f","gatewayName":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:58:44Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-d6591f","deploymentName":"maas-api-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:58:44Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-d6591f","deploymentName":"maas-api-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:58:44Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-d6591f","deploymentName":"maas-api-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:58:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:58:45Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-d6591f","deploymentName":"maas-api-e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T14:58:45Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"ecda381a-9b3e-44c1-87ee-228fc955146a","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T14:58:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"f8ec390b-ad32-4ab5-a728-b268fcdc8e47","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"f8ec390b-ad32-4ab5-a728-b268fcdc8e47","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T14:58:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"8e4292c1-0ea4-423a-b8b1-e7760afb7b0c","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"8e4292c1-0ea4-423a-b8b1-e7760afb7b0c","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:58:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:58:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T14:58:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"5e1f5086-6fb8-4db3-87d2-b33dd4175574","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"5e1f5086-6fb8-4db3-87d2-b33dd4175574","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T14:58:52Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"11247ee5-0bd3-41f9-a965-53e560eeddec","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:52Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"11247ee5-0bd3-41f9-a965-53e560eeddec","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T14:58:52Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"c2737c5b-e54d-402f-98ec-cfe3ef25aec6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:52Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"c2737c5b-e54d-402f-98ec-cfe3ef25aec6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:58:52Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"c2737c5b-e54d-402f-98ec-cfe3ef25aec6","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","tenantID":"e2e-inf-b-b132ed","subscriptionNamespace":"ai-tenant-e2e-inf-b-b132ed","gatewayName":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:58:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"468c45c0-5163-4262-b0e1-526f0fa24f63","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:58:52Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-b132ed","deploymentName":"maas-api-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:58:52Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-b132ed","deploymentName":"maas-api-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:58:52Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-b132ed","deploymentName":"maas-api-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:58:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"6a934b87-e93a-4878-9db0-dec672ef2147","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:58:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:58:53Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-b132ed","deploymentName":"maas-api-e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T14:58:53Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"c2737c5b-e54d-402f-98ec-cfe3ef25aec6","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T14:58:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"294b7599-1d57-4595-94ff-38fc50ad1d1e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"294b7599-1d57-4595-94ff-38fc50ad1d1e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:58:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T14:58:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"271c8ddc-9186-422d-8455-113f007eed4b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:58:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"271c8ddc-9186-422d-8455-113f007eed4b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"fe28a2d1-c329-40a3-9e10-be4bb48cbea8","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"6590c065-60c3-493e-9c61-60b7bed2711d","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"error","ts":"2026-07-13T14:59:02Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"6590c065-60c3-493e-9c61-60b7bed2711d","name":"test-model-d6591f","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"test-model-d6591f\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"c66ea038-4823-41e8-b64a-f97a23575ac5","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"993f38eb-048b-4a33-84f7-e402dee6ed59","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"error","ts":"2026-07-13T14:59:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"11cc2911-e8a1-49ef-bae0-5e0e64bc5766","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"test-model-d6591f-sub\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"0bcbcf38-7841-425a-a73d-957ce21041a8","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"2b754f79-2a95-4ad7-ba10-6cca4b53bc22","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"test-model-d6591f","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-inf-a-d6591f/test-model-d6591f-sub"]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"b4974013-9103-4a70-a994-f3ac2d0b2a23","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"b84eb1d0-57d8-42d8-9847-258676d092ea","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"be410717-c7d9-4422-aaa8-edfd9b25cbc4","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"622e859e-5ff1-4d00-b4c9-496ae65a5d7e","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"86f2afcb-7e6f-468a-b0a8-b4d572ab92be","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-d6591f","tenantID":"e2e-inf-a-d6591f","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-13T14:59:02Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"622e859e-5ff1-4d00-b4c9-496ae65a5d7e","name":"test-model-d6591f","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"test-model-d6591f\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"cb8918ee-0bc3-4966-982f-3d8d32f4ab43","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"86f2afcb-7e6f-468a-b0a8-b4d572ab92be"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"86f2afcb-7e6f-468a-b0a8-b4d572ab92be","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"e2e-inf-a-d6591f-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:59:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"9ffc537d-8fc6-42f3-82b7-7ac67f86cee6","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"5587ccef-487f-476e-af79-8ba95b2bd2ff","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-d6591f","tenantID":"e2e-inf-a-d6591f","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"5587ccef-487f-476e-af79-8ba95b2bd2ff"}
{"level":"info","ts":"2026-07-13T14:59:02Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"5587ccef-487f-476e-af79-8ba95b2bd2ff","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"e2e-inf-a-d6591f-maas-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-13T14:59:03Z","msg":"failed to update MaaSAuthPolicy status","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"5587ccef-487f-476e-af79-8ba95b2bd2ff","name":"test-model-d6591f-auth","error":"Operation cannot be fulfilled on maasauthpolicies.maas.opendatahub.io \"test-model-d6591f-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSAuthPolicyReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasauthpolicy_controller.go:1770\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSAuthPolicyReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasauthpolicy_controller.go:555\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"78eb8717-1d68-43b5-bb9e-22b68bcf3909","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"d93bd284-dffc-4990-a55d-5964929dfdb2","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:10Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:59:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"fd8dac43-4a34-4dda-a160-a3da2e86d2bc","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:10Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"d9598841-7050-45c4-bf8f-d0cfd25ae705","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-d6591f","tenantID":"e2e-inf-a-d6591f","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"75642e44-bfe2-4123-899a-00608c4c5352","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:10Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"d9598841-7050-45c4-bf8f-d0cfd25ae705"}
{"level":"info","ts":"2026-07-13T14:59:10Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"d9598841-7050-45c4-bf8f-d0cfd25ae705","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"e2e-inf-a-d6591f-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:59:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:59:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:59:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"bf0e78ac-176a-465d-acb4-2632089cb871","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"0eca2082-77dd-4a99-91fe-12ecbfccf232","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"3626990e-ba78-41f1-a373-2bd9c9ad30f5","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"74d67af8-b90c-4717-a8d6-c27a9a1dcd5e","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:22Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T14:59:22Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"8dd069de-9be0-45af-9386-893e46e6fd4e","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-d6591f","tenantID":"e2e-inf-a-d6591f","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:22Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"2c96dd37-a888-4f9d-be56-3432c3a0d05c","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:22Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"8dd069de-9be0-45af-9386-893e46e6fd4e"}
{"level":"info","ts":"2026-07-13T14:59:22Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"8dd069de-9be0-45af-9386-893e46e6fd4e","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"e2e-inf-a-d6591f-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:59:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T14:59:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"805645b0-66b3-44e3-8f52-a2c9a55b48f4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:59:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"805645b0-66b3-44e3-8f52-a2c9a55b48f4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T14:59:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"32bb2198-9a3c-42f9-bca3-836c61e03789","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:59:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"32bb2198-9a3c-42f9-bca3-836c61e03789","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T14:59:38Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"466a9dae-bdcd-4774-a454-0fc37865bd0c","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T14:59:38Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"466a9dae-bdcd-4774-a454-0fc37865bd0c","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"debfe3e1-c368-4d15-a001-3623b1051dfb","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"6c867ef0-63e5-49af-b83a-32c4c2d45b4f","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"9e292e3a-0a48-4c60-b9b5-d29ac673d65c","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"error","ts":"2026-07-13T14:59:39Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"22ca693d-2b84-47e0-ade4-df97e8e71506","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"test-model-b132ed-sub\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"d64c9a83-d050-4b11-b8e7-bfd082143361","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"464c6263-c773-4e51-a9f5-cd7a572e93bd","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"test-model-b132ed","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-inf-b-b132ed/test-model-b132ed-sub"]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"d3ed5e22-7e6c-409f-9021-e42370498cb5","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"328922e9-9666-466d-a0a0-52c189d596c4","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"40c1a845-ccaa-48cf-b451-2fd78e139f47","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"a94923c2-0f48-4572-b0dc-19232d2b6659","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"f08b8658-0ef1-4fcb-af51-5459b62e2346","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-b132ed","tenantID":"e2e-inf-b-b132ed","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"85f85712-f9cc-4a8f-9899-0c8e1a9409f9","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"f08b8658-0ef1-4fcb-af51-5459b62e2346"}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"f08b8658-0ef1-4fcb-af51-5459b62e2346","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"e2e-inf-b-b132ed-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:59:39Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"08d8af14-09f0-453c-822a-ff9987ea6336","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"4acc676d-1150-4c1f-bbd1-553b64fdac6d","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-b132ed","tenantID":"e2e-inf-b-b132ed","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"4acc676d-1150-4c1f-bbd1-553b64fdac6d"}
{"level":"info","ts":"2026-07-13T14:59:39Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"4acc676d-1150-4c1f-bbd1-553b64fdac6d","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"e2e-inf-b-b132ed-maas-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-13T14:59:39Z","msg":"failed to update MaaSAuthPolicy status","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"4acc676d-1150-4c1f-bbd1-553b64fdac6d","name":"test-model-b132ed-auth","error":"Operation cannot be fulfilled on maasauthpolicies.maas.opendatahub.io \"test-model-b132ed-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSAuthPolicyReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasauthpolicy_controller.go:1770\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSAuthPolicyReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasauthpolicy_controller.go:555\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T14:59:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"69de3976-1c38-42f3-a214-a548515b1658","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"31645cda-629d-4938-b2fc-77a2bb3e9e63","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5a81c2bf-80cb-4c18-a1bf-72214d92f19f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"f8f2525e-bf5d-4da6-b138-00b9b8c32e6e","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:42Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"766aaec4-a744-40ba-904a-cd7619080050","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-b132ed","tenantID":"e2e-inf-b-b132ed","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"97eda6e8-52d8-428f-964a-82143d5ae14a","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"766aaec4-a744-40ba-904a-cd7619080050"}
{"level":"info","ts":"2026-07-13T14:59:42Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"766aaec4-a744-40ba-904a-cd7619080050","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"e2e-inf-b-b132ed-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:59:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:59:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T14:59:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"bb28435c-f587-46a5-a655-3bda330d2fe5","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"ba5744db-08c3-46b9-976f-483ed92344c9","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"e52c8fb6-a7cb-4781-b973-3898ed866f8d","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:59:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"bf5fd516-2782-41a8-9f55-d96ccf17c167","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-b132ed","tenantID":"e2e-inf-b-b132ed","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"b382df30-6692-4984-897a-81e07436db1f","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"bf5fd516-2782-41a8-9f55-d96ccf17c167"}
{"level":"info","ts":"2026-07-13T14:59:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"bf5fd516-2782-41a8-9f55-d96ccf17c167","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"e2e-inf-b-b132ed-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T14:59:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"0cdd75b6-506e-4ea2-8672-5d05faa6d8c6","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T14:59:56Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T14:59:56Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"cd63356f-e801-4d53-a4fc-46c2db427505","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-b132ed","tenantID":"e2e-inf-b-b132ed","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T14:59:56Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"cf48926b-5940-4a5c-91d1-a8f9ed6193a9","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T14:59:56Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"cd63356f-e801-4d53-a4fc-46c2db427505"}
{"level":"info","ts":"2026-07-13T14:59:56Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"cd63356f-e801-4d53-a4fc-46c2db427505","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"e2e-inf-b-b132ed-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T15:00:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"de6f083c-b5fb-4a8b-b6ec-b660b07b80b7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"e319b6e1-2d59-4784-b05c-aa9b8a0f5cb1","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"c5889007-75e9-4583-b0c8-c4375560b6c9","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"d691ccfa-8acd-4bdc-9551-90d8d7321abf","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:04Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-auth","reconcileID":"68d898e1-39cf-4e04-87f3-127b524f8a25","MaaSAuthPolicy":{"name":"test-model-d6591f-auth","namespace":"ai-tenant-e2e-inf-a-d6591f"},"model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"dc0e8661-ba5a-4c14-96f3-965326f32cdb","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"29a9ec06-60b3-471b-978d-751b1d37a2fa","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"03a15ab8-cd1f-4d59-ae58-879514a71d04","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"c29bf65e-86bc-4fe6-8276-dbe2ec7c287f","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"2af33d97-597e-4ca9-8c94-19c28a85a538","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f","subscription":"test-model-d6591f-sub"}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"7d135b7a-9fe2-42d9-84c4-3d124848f8e3","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"2af33d97-597e-4ca9-8c94-19c28a85a538","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f-sub","reconcileID":"2af33d97-597e-4ca9-8c94-19c28a85a538","MaaSSubscription":{"name":"test-model-d6591f-sub","namespace":"ai-tenant-e2e-inf-a-d6591f"},"name":"maas-trlp-test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f","model":"ai-tenant-e2e-inf-a-d6591f/test-model-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"test-model-d6591f","reconcileID":"dc6d2a54-eaf4-4744-81b8-3f7f1bdf330c","MaaSModelRef":{"name":"test-model-d6591f","namespace":"ai-tenant-e2e-inf-a-d6591f"},"routeName":"test-model-d6591f-kserve-route","namespace":"ai-tenant-e2e-inf-a-d6591f","llmisvcName":"test-model-d6591f","gateway":"openshift-ingress/e2e-inf-a-d6591f","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-auth","reconcileID":"5285b0c6-25b1-4a9d-b64a-70b55dc3f6c1","MaaSAuthPolicy":{"name":"test-model-b132ed-auth","namespace":"ai-tenant-e2e-inf-b-b132ed"},"model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"7bbfd2b9-e47f-4efe-9817-0dfd2971597a","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:05Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"f1378d34-9ea6-4282-99e8-1f4a3a4b60f2","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"307e3075-88ea-4c42-99b4-0efe2dc4493a","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"ca2fdd01-d64c-4143-8906-02fca2fd578e","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"0f7d950e-513c-408a-9b6d-8f9a667fea40","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed","subscription":"test-model-b132ed-sub"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"4d7448a6-0509-497b-b1de-1f373313b418","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"0f7d950e-513c-408a-9b6d-8f9a667fea40","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed-sub","reconcileID":"0f7d950e-513c-408a-9b6d-8f9a667fea40","MaaSSubscription":{"name":"test-model-b132ed-sub","namespace":"ai-tenant-e2e-inf-b-b132ed"},"name":"maas-trlp-test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed","model":"ai-tenant-e2e-inf-b-b132ed/test-model-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"test-model-b132ed","reconcileID":"86565d7e-c088-4730-9b7d-5236e87c1af6","MaaSModelRef":{"name":"test-model-b132ed","namespace":"ai-tenant-e2e-inf-b-b132ed"},"routeName":"test-model-b132ed-kserve-route","namespace":"ai-tenant-e2e-inf-b-b132ed","llmisvcName":"test-model-b132ed","gateway":"openshift-ingress/e2e-inf-b-b132ed","hostnames":[]}
{"level":"info","ts":"2026-07-13T15:00:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"error","ts":"2026-07-13T15:00:06Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"3615f832-a570-4b6d-bb1a-4eb96fe611ee","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:00:06Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"3615f832-a570-4b6d-bb1a-4eb96fe611ee","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:00:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","namespace":"ai-tenant-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","count":0}
{"level":"info","ts":"2026-07-13T15:00:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-d6591f/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-d6591f","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-d6591f","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-d6591f"},"tenantIdentifier":"e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","tenant":"e2e-inf-a-d6591f","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T15:00:06Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-d6591f","deploymentName":"maas-api-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:06Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","gvk":"Deployment","name":"maas-api-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:07Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","gvk":"Service","name":"maas-api-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:07Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:07Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-d6591f"},"namespace":"ai-tenant-e2e-inf-a-d6591f","name":"default-tenant","reconcileID":"6f000d66-1e4e-48a0-aa8f-0cb95d7f8e7f","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-a-d6591f"}
{"level":"info","ts":"2026-07-13T15:00:13Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a064ff6e-9ffc-460d-82df-6e0ef2e98615","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:14Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"aa8b32f0-6267-45a8-97bf-f0bce6a90c76","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:00:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:00:18Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"b1eeee7c-74f8-4e5d-ad8a-57e969c13b6f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"3d126271-8999-4708-9f1e-ee0b04d062f9","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T15:00:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"56562b2c-7cf4-487c-9860-d06e11853b17","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:00:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"56562b2c-7cf4-487c-9860-d06e11853b17","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:00:23Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T15:00:23Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"af0c5e80-84d2-435b-b388-537e96637434","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:00:23Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"af0c5e80-84d2-435b-b388-537e96637434","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:00:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:00:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:00:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"error","ts":"2026-07-13T15:00:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"6c99e565-68a4-4965-9d38-088b08208046","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:00:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"6c99e565-68a4-4965-9d38-088b08208046","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:00:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","namespace":"ai-tenant-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","count":0}
{"level":"info","ts":"2026-07-13T15:00:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-b132ed/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-b132ed","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-b132ed","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-b132ed"},"tenantIdentifier":"e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","tenant":"e2e-inf-b-b132ed","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T15:00:50Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-b132ed","deploymentName":"maas-api-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","gvk":"Deployment","name":"maas-api-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","gvk":"Service","name":"maas-api-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:50Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b132ed"},"namespace":"ai-tenant-e2e-inf-b-b132ed","name":"default-tenant","reconcileID":"5af25aae-da21-4b59-af4f-c612b052e5f3","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-b-b132ed"}
{"level":"info","ts":"2026-07-13T15:00:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"315be3a3-bb8c-4fa6-8649-f183664254fa","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"478966da-8f03-4a70-ba68-83770066e7d3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:59Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a8b09070-83de-4698-8cf2-a61e742a3031","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:00:59Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"db9d1584-adb8-4154-9107-4fd97bd92c77","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T15:01:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:01:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"d34507a9-12b5-41cd-aa90-e5a576d7fa04","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:01:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"d34507a9-12b5-41cd-aa90-e5a576d7fa04","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:01:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:01:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"5a5fb123-d85a-4a56-89e7-f05738ee4b73","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:01:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"5a5fb123-d85a-4a56-89e7-f05738ee4b73","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:01:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:01:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"53281d02-075a-4b4f-a005-db1f5ee27317","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:01:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"53281d02-075a-4b4f-a005-db1f5ee27317","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:01:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T15:01:05Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6f6047cf-a4c6-43ac-ba57-32046424a8aa","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:01:05Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6f6047cf-a4c6-43ac-ba57-32046424a8aa","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:01:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:01:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:01:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:01:47Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"f082b442-58a0-467c-b62e-2c9809033cde","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:01:47Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"f082b442-58a0-467c-b62e-2c9809033cde","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:01:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:01:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:01:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T15:01:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"74ef96d0-d13c-489c-a8b0-3e9a68c620f2","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:01:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"74ef96d0-d13c-489c-a8b0-3e9a68c620f2","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:02:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:02:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:02:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:02:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"6f4de974-db7f-44d0-b9d3-564173cd9868","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:02:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"6f4de974-db7f-44d0-b9d3-564173cd9868","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:02:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T15:02:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7944cb3f-c375-405a-8445-420cfd9774a6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:02:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7944cb3f-c375-405a-8445-420cfd9774a6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:02:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:02:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:03:17Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:03:17Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"53f90afe-2178-4267-90bf-870a6d6efdee","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:03:17Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"53f90afe-2178-4267-90bf-870a6d6efdee","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:03:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:03:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:03:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T15:03:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"3473d62b-2c87-4246-b392-ded5ec932b10","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:03:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"3473d62b-2c87-4246-b392-ded5ec932b10","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:03:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:03:48Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:04:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:04:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"b42d6dab-528d-47d4-9d46-f92dfcca1be7","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:04:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"b42d6dab-528d-47d4-9d46-f92dfcca1be7","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:04:06Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T15:04:06Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ba297fcb-0dfa-4cfe-ba43-b8730daa85c2","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:04:06Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ba297fcb-0dfa-4cfe-ba43-b8730daa85c2","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:04:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"error","ts":"2026-07-13T15:04:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"3c01d848-d80e-4ce4-9e9c-ec1114a6742e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T15:04:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"3c01d848-d80e-4ce4-9e9c-ec1114a6742e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T15:04:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","namespace":"ai-tenant-e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","count":0}
{"level":"info","ts":"2026-07-13T15:04:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-93710c/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-93710c","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-93710c","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-93710c"},"tenantIdentifier":"e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","tenant":"e2e-api-a-93710c","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T15:04:11Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-api-a-93710c","deploymentName":"maas-api-e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","gvk":"Deployment","name":"maas-api-e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","gvk":"Service","name":"maas-api-e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","gvk":"HTTPRoute","name":"maas-api-route-e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-93710c"},"namespace":"ai-tenant-e2e-api-a-93710c","name":"default-tenant","reconcileID":"607857ea-aa79-4c58-a447-1bfada486ea0","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-api-a-93710c"}
{"level":"info","ts":"2026-07-13T15:04:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T15:04:18Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}