{"level":"info","ts":"2026-07-13T20:08:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:08:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:09:11Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:09:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"e1fd1261-dca7-4998-9d14-d6490464e37b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:09:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"e1fd1261-dca7-4998-9d14-d6490464e37b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:09:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:09:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:09:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:09:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:09:56Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:09:56Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"5b0a688b-f76d-4224-8977-9510a20d92a2","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:09:56Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"5b0a688b-f76d-4224-8977-9510a20d92a2","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:10:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:10:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:10:41Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:10:41Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"193b4a29-480c-4099-853c-74c2e4ccc522","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:10:41Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"193b4a29-480c-4099-853c-74c2e4ccc522","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:10:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:10:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:11:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:11:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:11:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:11:26Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0e5ad8d2-0f07-4687-8649-02981c00f616","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:26Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0e5ad8d2-0f07-4687-8649-02981c00f616","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"error","ts":"2026-07-13T20:11:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"2ead4590-de5f-46c1-a3ea-51dfa344a62e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"2ead4590-de5f-46c1-a3ea-51dfa344a62e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"error","ts":"2026-07-13T20:11:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"30dcaf49-a798-4c05-b6fe-d55ace6f4f72","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"30dcaf49-a798-4c05-b6fe-d55ace6f4f72","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"error","ts":"2026-07-13T20:11:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"9296c960-b210-476a-bebd-6ce0e2e205cd","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"9296c960-b210-476a-bebd-6ce0e2e205cd","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"error","ts":"2026-07-13T20:11:37Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"b5e85dd2-1f40-49d0-8a77-f28092ffa89f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:37Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"b5e85dd2-1f40-49d0-8a77-f28092ffa89f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","namespace":"ai-tenant-e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","count":0}
{"level":"info","ts":"2026-07-13T20:11:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-3b029816/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-3b029816","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-3b029816","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-3b029816"},"tenantIdentifier":"e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","tenant":"e2e-ait-3b029816","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T20:11:37Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-ait-3b029816","deploymentName":"maas-api-e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","gvk":"Deployment","name":"maas-api-e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","gvk":"Service","name":"maas-api-e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","gvk":"HTTPRoute","name":"maas-api-route-e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:37Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-3b029816"},"namespace":"ai-tenant-e2e-ait-3b029816","name":"default-tenant","reconcileID":"59f82972-cb3e-4b6c-9123-955c339ac9d8","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-ait-3b029816"}
{"level":"info","ts":"2026-07-13T20:11:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"error","ts":"2026-07-13T20:11:44Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"839409e7-c9f6-4072-a55f-8d6cd1bdea29","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:44Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"839409e7-c9f6-4072-a55f-8d6cd1bdea29","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"error","ts":"2026-07-13T20:11:44Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"adec6af9-19ae-4e38-97cd-136f73e68e03","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:44Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"adec6af9-19ae-4e38-97cd-136f73e68e03","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"error","ts":"2026-07-13T20:11:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"3c72c98c-14be-48dd-aa5a-1d7f76bbe9bb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"3c72c98c-14be-48dd-aa5a-1d7f76bbe9bb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"error","ts":"2026-07-13T20:11:49Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"3db563b4-e3ff-44e2-9b57-ce80ea058c23","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:49Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"3db563b4-e3ff-44e2-9b57-ce80ea058c23","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","namespace":"ai-tenant-e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","count":0}
{"level":"info","ts":"2026-07-13T20:11:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-d40c5666/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-d40c5666","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-d40c5666","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-d40c5666"},"tenantIdentifier":"e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","tenant":"e2e-ait-d40c5666","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T20:11:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-ait-d40c5666","deploymentName":"maas-api-e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","gvk":"Deployment","name":"maas-api-e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","gvk":"Service","name":"maas-api-e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","gvk":"HTTPRoute","name":"maas-api-route-e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:49Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-d40c5666"},"namespace":"ai-tenant-e2e-ait-d40c5666","name":"default-tenant","reconcileID":"16910307-0fa7-4a55-bf2f-66bf3f907c8a","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-ait-d40c5666"}
{"level":"info","ts":"2026-07-13T20:11:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:11:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:11:58Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"error","ts":"2026-07-13T20:11:58Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"356cdde6-5fe2-44f6-80df-c92dc206be0d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:58Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"356cdde6-5fe2-44f6-80df-c92dc206be0d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:58Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"error","ts":"2026-07-13T20:11:58Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"f093d16f-81cd-4426-9772-2656b520aa87","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:58Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"f093d16f-81cd-4426-9772-2656b520aa87","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:58Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:11:58Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"f093d16f-81cd-4426-9772-2656b520aa87","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","tenantID":"e2e-derive-5d9c011d","subscriptionNamespace":"ai-tenant-e2e-derive-5d9c011d","gatewayName":"e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:11:58Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-5d9c011d","deploymentName":"maas-api-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:11:58Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-5d9c011d","deploymentName":"maas-api-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:11:58Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-5d9c011d","deploymentName":"maas-api-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:11:59Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:11:59Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-5d9c011d","deploymentName":"maas-api-e2e-derive-5d9c011d"}
{"level":"error","ts":"2026-07-13T20:11:59Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"f093d16f-81cd-4426-9772-2656b520aa87","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:59Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"error","ts":"2026-07-13T20:11:59Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"f31c8cf4-40d1-45e4-a25f-89592c6e5418","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:59Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"f31c8cf4-40d1-45e4-a25f-89592c6e5418","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:11:59Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"error","ts":"2026-07-13T20:11:59Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"a9e2e809-e6a5-45db-973a-a4bf4713db63","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:11:59Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"a9e2e809-e6a5-45db-973a-a4bf4713db63","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:12:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"cc9e82e2-2d9c-48f5-b367-63067346007b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:12:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ee3fff9a-89cd-49d9-84ae-d88b099b2ae9","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:12:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"error","ts":"2026-07-13T20:12:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"8cdc73bc-524b-4458-9765-bf51fbed65bb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:12:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"8cdc73bc-524b-4458-9765-bf51fbed65bb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:12:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","namespace":"ai-tenant-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","count":0}
{"level":"info","ts":"2026-07-13T20:12:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-5d9c011d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-5d9c011d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-5d9c011d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-5d9c011d"},"tenantIdentifier":"e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","tenant":"e2e-derive-5d9c011d","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T20:12:02Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-5d9c011d","deploymentName":"maas-api-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","gvk":"Deployment","name":"maas-api-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","gvk":"Service","name":"maas-api-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","gvk":"HTTPRoute","name":"maas-api-route-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-5d9c011d"},"namespace":"ai-tenant-e2e-derive-5d9c011d","name":"default-tenant","reconcileID":"373ebbdb-c3ca-49bf-a0ee-be21f94c05d8","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-derive-5d9c011d"}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ba8b982d-435c-4337-b1b9-de450552c39e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f3ff8ae8-8cc1-4840-b9e7-2ec1324fbcde","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:09Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-8ecb56c9/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-policy-8ecb56c9","reconcileID":"bdcf7e50-fd20-4124-9c79-dce53dfde704","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-policy-8ecb56c9","reconcileID":"bdcf7e50-fd20-4124-9c79-dce53dfde704"}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-policy-8ecb56c9","reconcileID":"bdcf7e50-fd20-4124-9c79-dce53dfde704","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a64705bd-863c-44f2-ba44-f6e4d67d73eb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"70a1546a-081a-481f-9c1c-4806726f76c9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ccd44167-726a-4bf6-8509-cd145e4ae4e6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-sub-8ecb56c9","reconcileID":"8ac31086-830e-4a48-8218-64a317d212b8","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-8ecb56c9/e2e-sub-8ecb56c9","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c6763107-5619-4cc8-86ce-adfd78bdc497","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"7f381ee2-d833-4dbe-9ca1-426964169e17","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:12:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-sub-8ecb56c9","reconcileID":"c70d2c7e-3372-4ea1-9014-5747894f49e2","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-policy-8ecb56c9","reconcileID":"f52bf8f0-019b-47f6-858e-8cf333469f34","MaaSAuthPolicy":{"name":"e2e-policy-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dccf3ff1-b6c5-4b2d-9150-6b9c9cf50b74","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4ed5cc11-29e8-46b3-a223-dfc88fee90fc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-sub-8ecb56c9","reconcileID":"ea71b4d1-f2a4-4ab4-986e-d279d4514268","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-sub-8ecb56c9"}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d5dde2ba-3136-4d3a-9ec7-4b1d8d6177f3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"namespace":"ai-tenant-e2e-mt-8ecb56c9","name":"e2e-sub-8ecb56c9","reconcileID":"ea71b4d1-f2a4-4ab4-986e-d279d4514268","MaaSSubscription":{"name":"e2e-sub-8ecb56c9","namespace":"ai-tenant-e2e-mt-8ecb56c9"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1bc1e615-2f79-435e-bc36-36c2976d7feb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"dc2b1101-eb1b-4332-9f71-594748f86980","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:12:11Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:12:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"40633383-4e36-4d86-844a-3a8663ca27a3","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:12:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"40633383-4e36-4d86-844a-3a8663ca27a3","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:12:15Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"1fc44f43-baad-468d-bc64-517099268e40","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:12:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"02f207b2-ad83-4f0c-985c-d1aaba569efb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"98781076-dc47-4801-a397-5bf66f24a3b1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:17Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-8c33c7f2/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:12:17Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"namespace":"ai-tenant-e2e-mt-8c33c7f2","name":"e2e-policy-8c33c7f2","reconcileID":"c96c22e3-1c4e-42b9-a32c-bd5d8fa05e4c","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:12:17Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"namespace":"ai-tenant-e2e-mt-8c33c7f2","name":"e2e-policy-8c33c7f2","reconcileID":"c96c22e3-1c4e-42b9-a32c-bd5d8fa05e4c"}
{"level":"info","ts":"2026-07-13T20:12:17Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"namespace":"ai-tenant-e2e-mt-8c33c7f2","name":"e2e-policy-8c33c7f2","reconcileID":"c96c22e3-1c4e-42b9-a32c-bd5d8fa05e4c","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:12:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"be923330-0053-406e-9828-20b0681b8906","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:12:20Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"1faeec42-fcdf-4674-98e2-8926daa67cb5","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:12:21Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"320daafb-06b0-4969-8ebd-dd4cbd346067","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:12:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:12:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:12:43Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"namespace":"ai-tenant-e2e-mt-8c33c7f2","name":"e2e-policy-8c33c7f2","reconcileID":"d9614cb8-7e25-49e4-a52d-cc2051707036","MaaSAuthPolicy":{"name":"e2e-policy-8c33c7f2","namespace":"ai-tenant-e2e-mt-8c33c7f2"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:12:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:12:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:12:56Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:12:56Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"99d2798d-93c9-409f-82a9-d5d5df74c22e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:12:56Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"99d2798d-93c9-409f-82a9-d5d5df74c22e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:23Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-da7c510c/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f82e88fc-f3b1-4541-a056-fd0418bacbf4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"namespace":"ai-tenant-e2e-mt-da7c510c","name":"e2e-policy-da7c510c","reconcileID":"5ec10a9e-f43e-4489-98ce-e127b772a412","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"namespace":"ai-tenant-e2e-mt-da7c510c","name":"e2e-policy-da7c510c","reconcileID":"5ec10a9e-f43e-4489-98ce-e127b772a412"}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"namespace":"ai-tenant-e2e-mt-da7c510c","name":"e2e-policy-da7c510c","reconcileID":"5ec10a9e-f43e-4489-98ce-e127b772a412","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fde00d7f-a330-45dc-9968-7c65a1054218","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"namespace":"ai-tenant-e2e-mt-da7c510c","name":"e2e-policy-da7c510c","reconcileID":"b8702763-f3f1-4930-b21a-529b2ef40dcc","MaaSAuthPolicy":{"name":"e2e-policy-da7c510c","namespace":"ai-tenant-e2e-mt-da7c510c"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dfcc96fc-0849-4f22-a77c-56faeadaa43e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"84e4fda4-d22f-40e6-b6a4-10c25ba47dff","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:13:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"537a7ec6-f37d-4bb0-a129-d797fa2b95c3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3091c4e3-1620-431b-8766-d988b1cf7047","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:30Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-6bc0968e/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-policy-6bc0968e","reconcileID":"621cc513-d8ca-416a-a5e3-5fc58b5ff1c6","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-policy-6bc0968e","reconcileID":"621cc513-d8ca-416a-a5e3-5fc58b5ff1c6"}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-policy-6bc0968e","reconcileID":"621cc513-d8ca-416a-a5e3-5fc58b5ff1c6","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"68d2d172-5114-40d5-ab28-dd43027ec7c9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"34c57232-9ffa-4c53-80bc-0b33afcdb606","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"51981faf-7a46-46e1-94db-da9869cc98ee","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T20:13:30Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-sub-6bc0968e","reconcileID":"e9c7299c-9872-4c42-8e98-49667e3aea63","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-sub-6bc0968e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c08b9de8-37f0-44e4-9f13-a54b14c70486","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-sub-6bc0968e","reconcileID":"25478b53-d800-433f-a2da-190e23583d35","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-6bc0968e/e2e-shared-sub-6bc0968e","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b378a69e-7b55-4c5b-8f84-3532bc269795","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"50f2340e-b255-47a5-8600-8d3da415e6b1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:13:30Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-sub-6bc0968e","reconcileID":"6da685c0-b00f-4903-969b-006b93cd16f6","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"112ec251-a6ed-4943-b430-d562f0ee03d6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:31Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-2c3c8c5e/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"32f35f55-e298-4318-90f3-9ad2e41adb61","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-policy-6bc0968e","reconcileID":"9d87df66-4ee2-4141-9459-0453871f4736","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-policy-6bc0968e","reconcileID":"9d87df66-4ee2-4141-9459-0453871f4736"}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-policy-6bc0968e","reconcileID":"9d87df66-4ee2-4141-9459-0453871f4736","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b3513051-e8de-4ebb-b1bd-33501e9edfa5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"73d1f464-2434-478f-95dc-e91dec9d5889","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"77daa28c-073e-400c-b12f-aaa25794518f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T20:13:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"9d5a0f9a-3dd3-436a-a4b8-19b480e6da93","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-sub-6bc0968e\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"552aa02b-8046-436d-ae38-ebd31d8f4db9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"ef99119e-248f-44da-90b9-56d5abc175a1","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-2c3c8c5e/e2e-shared-sub-6bc0968e","ai-tenant-e2e-mt-6bc0968e/e2e-shared-sub-6bc0968e","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"245ae7fc-483f-41e2-8abc-f5a5b32795e5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"26c24b66-dc02-42f8-aa58-1ef601361401","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-sub-6bc0968e","reconcileID":"0f9a51d4-89be-48b1-bce0-317fd721c99d","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T20:13:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"cdad8841-80d7-4370-b0fa-03d6bbf8e747","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-policy-6bc0968e","reconcileID":"61559897-977d-466c-9f81-5ed6bfe84dfa","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bbbc4819-c205-481c-a78b-8210fca95127","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"af427f87-f8b6-4f99-befc-da1d47357aa0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-sub-6bc0968e","reconcileID":"1e1a47cb-8ada-4396-b3fc-0c449f63b8cf","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-sub-6bc0968e"}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"02f5f348-9121-4f38-bc27-1ffe0afcc806","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"namespace":"ai-tenant-e2e-mt-6bc0968e","name":"e2e-shared-sub-6bc0968e","reconcileID":"1e1a47cb-8ada-4396-b3fc-0c449f63b8cf","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-6bc0968e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-2c3c8c5e/e2e-shared-sub-6bc0968e","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d9acf5b6-5704-4b85-b944-88105783cdcb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ae73d3f6-c5fc-4b40-b6e8-9d9d37ad431d","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:13:32Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"d99e4634-d894-4087-aabc-3071a2635e77","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:13:33Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"223dcad4-1253-4cfd-9bbf-1fb6a5cf747f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:13:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"8e9b4724-fdfa-4826-9921-5265459bc4f7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:13:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"6b8c8a6a-09c3-4b48-ac53-0a6bc250f947","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-policy-6bc0968e","reconcileID":"c2d47eee-d5ac-4b99-aef2-103d3ddebb43","MaaSAuthPolicy":{"name":"e2e-shared-policy-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2563814e-8309-4f32-99a7-a43e9c9f9f4d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"67df9a91-980d-4bf9-93d6-d05df42faa4e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"9b66fbc9-9bbd-44bf-ba64-fbf2f9f7d59b","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-sub-6bc0968e"}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"50c5562b-d944-46e3-909e-930480a6e5bc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"namespace":"ai-tenant-e2e-mt-2c3c8c5e","name":"e2e-shared-sub-6bc0968e","reconcileID":"9b66fbc9-9bbd-44bf-ba64-fbf2f9f7d59b","MaaSSubscription":{"name":"e2e-shared-sub-6bc0968e","namespace":"ai-tenant-e2e-mt-2c3c8c5e"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5097cb6b-f04c-450d-8531-a33b9ec79a2a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:13:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"9df41377-dd12-4c5b-8b12-5afd221e8fd8","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:13:41Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:13:41Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"58d0037d-590a-4a5a-8c56-6503531222cb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:13:41Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"58d0037d-590a-4a5a-8c56-6503531222cb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4385f530-a68f-4be2-9d97-885e6851d43a","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:13:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c8dfd847-aa83-44b1-8a5a-5f1ac9538dbd","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:13:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a9669fac-85c0-429a-baf0-fee177716f5f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:13:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"error","ts":"2026-07-13T20:13:49Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"ced82934-951d-4c62-9197-843893051662","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:13:49Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"ced82934-951d-4c62-9197-843893051662","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"error","ts":"2026-07-13T20:13:49Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"acc9fe29-2b0f-48f1-8056-2d890ab70e4a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:13:49Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"acc9fe29-2b0f-48f1-8056-2d890ab70e4a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:49Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:13:49Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"acc9fe29-2b0f-48f1-8056-2d890ab70e4a","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","tenantID":"e2e-mt-f915346b","subscriptionNamespace":"ai-tenant-e2e-mt-f915346b","gatewayName":"e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:13:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-f915346b","deploymentName":"maas-api-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:13:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-f915346b","deploymentName":"maas-api-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:13:49Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-f915346b","deploymentName":"maas-api-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:13:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:13:50Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-f915346b","deploymentName":"maas-api-e2e-mt-f915346b"}
{"level":"error","ts":"2026-07-13T20:13:50Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"acc9fe29-2b0f-48f1-8056-2d890ab70e4a","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"error","ts":"2026-07-13T20:13:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"5bd6bab1-d7d4-4d99-b556-ec72ca0bf084","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:13:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"5bd6bab1-d7d4-4d99-b556-ec72ca0bf084","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:50Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"error","ts":"2026-07-13T20:13:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"e39324c4-e081-49a8-8c78-97fc8723dbdf","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:13:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"e39324c4-e081-49a8-8c78-97fc8723dbdf","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:13:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:13:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:13:55Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0c13fd80-2931-4d11-b00d-c13e0e4b907a","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:13:55Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4111f176-750e-4d50-ac18-71b7906a84ef","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:00Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"7da6065a-1865-48c9-a801-f1be0dc20ae5","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"4fe9855c-13cc-43ae-a82b-1a8143f93631","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"error","ts":"2026-07-13T20:14:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"afa76dee-6505-49df-9449-806276bb655a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:14:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"afa76dee-6505-49df-9449-806276bb655a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:14:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","namespace":"ai-tenant-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","count":0}
{"level":"info","ts":"2026-07-13T20:14:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-f915346b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-f915346b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-f915346b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-f915346b"},"tenantIdentifier":"e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","tenant":"e2e-mt-f915346b","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T20:14:01Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-f915346b","deploymentName":"maas-api-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","gvk":"Deployment","name":"maas-api-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","gvk":"Service","name":"maas-api-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-f915346b"},"namespace":"ai-tenant-e2e-mt-f915346b","name":"default-tenant","reconcileID":"bfbaa085-6042-4591-984f-ee2ddeaff285","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-f915346b"}
{"level":"info","ts":"2026-07-13T20:14:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4bbe8f62-72e9-40a8-ac34-27f5fb680367","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"04ce8aeb-a05c-4833-b7fe-1e598e0717fa","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"d719f703-a095-44d1-9cc4-7f7b03d5bd8f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"40492f2a-fe2b-4f15-b641-746d79958554","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"17e178cf-cf33-4044-b36a-e30039cb1b4d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:22Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e4249b00-af2b-4733-9621-15944cfa4fad","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-ac027a4f","reconcileID":"22beb5bd-15d7-46c3-87fc-ef497f6b66d5","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-ac027a4f","reconcileID":"22beb5bd-15d7-46c3-87fc-ef497f6b66d5"}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-ac027a4f","reconcileID":"22beb5bd-15d7-46c3-87fc-ef497f6b66d5","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1e2136cc-1d40-470b-ab1d-ecb41c8b3a26","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-ac027a4f","reconcileID":"b6d21ac0-a621-4ee0-9630-4c22500f11e3","MaaSAuthPolicy":{"name":"e2e-default-ns-ac027a4f","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3f91b7b6-0151-47ce-a38b-e38d7b4fa9ae","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b78196d7-2499-4a49-8e9a-bdb2cf44b6e5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fb49ced9-6ed4-4cf7-9ce5-8113a6c02dfc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"181c6118-1257-4455-9d35-24218cade081","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:23Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"92f9ef62-99e0-4447-8651-6fefa10bff1e","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"92f9ef62-99e0-4447-8651-6fefa10bff1e"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"92f9ef62-99e0-4447-8651-6fefa10bff1e","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7cbb21e6-4830-4e71-b97f-7055311c57ce","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:23Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"4e3bde22-cd17-4d27-8fcc-4adf2cc31771","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"4e3bde22-cd17-4d27-8fcc-4adf2cc31771"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"4e3bde22-cd17-4d27-8fcc-4adf2cc31771","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-1d4ba4bb","reconcileID":"89267935-6f3b-42ea-89cd-9a870db26343","MaaSAuthPolicy":{"name":"e2e-gw-auth-1d4ba4bb","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ecca0cba-38a5-40b7-9bf4-5c11df8ea0fc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"cacded97-0d53-4e99-80d8-2b2e8b412298","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:14:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:14:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:14:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:14:26Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"56f61a6f-5a08-4a99-b14c-1e0abe05917d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:14:26Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"56f61a6f-5a08-4a99-b14c-1e0abe05917d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:14:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:14:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"7d7b1013-009d-41cc-b58c-1ebec21f00dd","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:14:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"7d7b1013-009d-41cc-b58c-1ebec21f00dd","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:14:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:14:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"94938174-0d1e-40eb-bc71-acf179e3f58a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:14:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"94938174-0d1e-40eb-bc71-acf179e3f58a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:14:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:14:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"03ed531f-42f5-4f02-864f-2c4825f24768","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:14:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"03ed531f-42f5-4f02-864f-2c4825f24768","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:14:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:14:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:15:11Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:15:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6501c36f-edef-4e47-8685-f16017f03b0f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:15:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6501c36f-edef-4e47-8685-f16017f03b0f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:15:17Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:15:17Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"94310924-a2de-4fb1-9247-fecade5d0fb5","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:15:17Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"94310924-a2de-4fb1-9247-fecade5d0fb5","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:15:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:15:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:15:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:15:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:15:56Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:15:56Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"46d88335-afb3-4e31-8256-ed4d80c4bde8","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:15:56Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"46d88335-afb3-4e31-8256-ed4d80c4bde8","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:16:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:16:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"027c9d48-5454-4dd9-901c-37c12c6df981","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:16:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"027c9d48-5454-4dd9-901c-37c12c6df981","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:16:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:16:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:16:41Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:16:41Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"78b2320d-6fb6-4428-8e2a-e04bf612d158","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:16:41Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"78b2320d-6fb6-4428-8e2a-e04bf612d158","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:16:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:16:47Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"2eeacb3a-ed5c-4959-9d9a-bd5566ed2e6e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:16:47Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"2eeacb3a-ed5c-4959-9d9a-bd5566ed2e6e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:16:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:16:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:17:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:17:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:17:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:17:26Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f6e14e85-0f3a-4be2-a866-04a15c9e8925","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:17:26Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f6e14e85-0f3a-4be2-a866-04a15c9e8925","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:17:32Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:17:32Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"b8327d5a-8b46-44df-a71c-7a4dce53581f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:17:32Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"b8327d5a-8b46-44df-a71c-7a4dce53581f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:17:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"error","ts":"2026-07-13T20:17:41Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"ec1a3408-7c94-40f9-8810-7a1255a3559a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:17:41Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"ec1a3408-7c94-40f9-8810-7a1255a3559a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:17:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","namespace":"ai-tenant-e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","count":0}
{"level":"info","ts":"2026-07-13T20:17:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-62c05fae/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-62c05fae","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-62c05fae","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-62c05fae"},"tenantIdentifier":"e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","tenant":"e2e-mt-62c05fae","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T20:17:41Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-62c05fae","deploymentName":"maas-api-e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","gvk":"Deployment","name":"maas-api-e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","gvk":"Service","name":"maas-api-e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:41Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-62c05fae"},"namespace":"ai-tenant-e2e-mt-62c05fae","name":"default-tenant","reconcileID":"73736abe-22be-49a8-85bd-ab28cb81841f","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-62c05fae"}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8a595e4f-7425-44a2-ae29-37a238b87e6d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:48Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b9891a1c-7cc9-430f-9f77-c21e1a195db1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-ce0b05","reconcileID":"695372c3-b540-43f6-92c7-4ea0bf42794f","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-ce0b05","reconcileID":"695372c3-b540-43f6-92c7-4ea0bf42794f"}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-ce0b05","reconcileID":"695372c3-b540-43f6-92c7-4ea0bf42794f","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"704ea62c-4e7e-420e-96f4-2732c2c17646","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0ce27229-38d8-456e-9d8e-6d9153bc1ff6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bc9fbbd6-f851-4596-acc1-80ec659d7092","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-ce0b05","reconcileID":"18f18bd3-c472-463b-ba53-10b1ae776b99","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-default-int-sub-ce0b05","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d03b90e6-e7c3-4acc-9a88-33c972ba8ef1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"1fed986f-541b-4857-986b-9d5614666cc2","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:17:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-ce0b05","reconcileID":"6d453139-3957-45dd-9576-a78bf6f50ddc","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-ce0b05","reconcileID":"b6dec67a-06b9-4f51-969d-00c11072ce32","MaaSAuthPolicy":{"name":"e2e-default-int-auth-ce0b05","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"48375e4b-ff3d-455a-b266-f495f0231aef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"95667951-2643-4c5c-9d15-7bc31ed70d4d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-ce0b05","reconcileID":"65e2f0c7-31d4-40ad-b1a0-6d3c866fce02","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-default-int-sub-ce0b05"}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3d4550bf-e801-4419-92bf-f23bd68f0fb2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-ce0b05","reconcileID":"65e2f0c7-31d4-40ad-b1a0-6d3c866fce02","MaaSSubscription":{"name":"e2e-default-int-sub-ce0b05","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"71cfcf6f-0038-4088-9071-48cb82a7bb89","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"03752d7a-3619-4790-9395-1c8f9e5611b1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6e6cedf6-5528-4f35-96c0-e56bc4ebd909","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-39057eed/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fa2bb901-f7c9-4c95-83ff-26e35e3d39e3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-policy-39057eed","reconcileID":"ed0914dc-ab37-438d-8dca-56ac314d70aa","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-policy-39057eed","reconcileID":"ed0914dc-ab37-438d-8dca-56ac314d70aa"}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-policy-39057eed","reconcileID":"ed0914dc-ab37-438d-8dca-56ac314d70aa","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ebde68cf-6968-4184-9806-899e3242cf49","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"176f4c25-df2b-4517-9b77-40627cf5725f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6fa6f8bd-feef-454a-97d5-c96391ca4ac9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-sub-39057eed","reconcileID":"086be7c5-5fbc-468b-b8e9-51c09c5edfd1","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-39057eed/e2e-shared-int-sub-39057eed","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6d29583e-31cf-47de-8d61-050351e055a2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ad9643cf-5330-4e8f-9870-b89964b31e71","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-sub-39057eed","reconcileID":"14e4d312-e7c9-4cea-9166-c8e12f0b8e97","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b5d403fc-0931-4604-9190-9d6eff7ab0bf","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-ea2ff266/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d6b2131b-509f-49b1-82c7-03ba66351b13","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-policy-39057eed","reconcileID":"b57e4e74-b875-49f1-b08f-22ae044fe63f","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-policy-39057eed","reconcileID":"b57e4e74-b875-49f1-b08f-22ae044fe63f"}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-policy-39057eed","reconcileID":"b57e4e74-b875-49f1-b08f-22ae044fe63f","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"57dcb2ea-8b0e-4b61-85d1-217fbd7638f5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"95f15567-6267-4a62-8c1c-3a14d54fe4e2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"26035d55-4163-4f2e-a19d-c8697282a5c9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T20:17:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-sub-39057eed","reconcileID":"ac83b3fb-13ad-40d5-8dba-5ec2eb5238c2","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-39057eed\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8db9d4ef-5660-49a5-9c11-7f20eb62e226","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9f329801-0759-408c-ac83-8701042080d6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-sub-39057eed","reconcileID":"6db44343-90cb-45e1-977a-d3a204d99126","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-39057eed/e2e-shared-int-sub-39057eed","ai-tenant-e2e-mt-ea2ff266/e2e-shared-int-sub-39057eed","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"39f75222-42da-4653-8eae-009a2a939c7b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"2780349f-4c65-4c37-b42b-419be66066ed","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-sub-39057eed","reconcileID":"daebfea2-ea7d-41be-ae36-bcafdbd94141","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T20:17:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-sub-39057eed","reconcileID":"f2296f2e-ddf0-4074-9340-fecbe066988b","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-policy-39057eed","reconcileID":"2bb01871-968b-4cd2-b9ce-e5f3e8129636","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d96cf4a2-612f-4f27-9d66-8f0cb2e874ed","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bacaffb9-dc99-4f95-a479-0e2ad3d17e68","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-sub-39057eed","reconcileID":"d07e8d46-797b-4297-9c91-0c9c5affb851","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-39057eed"}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4b8565f8-e11b-4453-ae4d-18110888de57","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"namespace":"ai-tenant-e2e-mt-39057eed","name":"e2e-shared-int-sub-39057eed","reconcileID":"d07e8d46-797b-4297-9c91-0c9c5affb851","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-39057eed"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-ea2ff266/e2e-shared-int-sub-39057eed","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1e3a430c-a86c-4869-8a32-3956ce4644f4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5ec736d0-1930-487f-bde6-fc7428954228","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:17:51Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-sub-39057eed","reconcileID":"54f458b8-d095-49ff-a593-195c493ee883","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T20:17:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:17:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:17:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0e946755-c858-4a2a-b08c-ed29f30ad59d","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-policy-39057eed","reconcileID":"01756387-1057-4aaa-b880-b6fb70512d69","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0c81a5cb-f566-44ca-bd0a-6d21a64e5cd7","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b8ab6752-245f-446c-8872-2a7ddef2e43e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-sub-39057eed","reconcileID":"93d2225b-26c7-407c-9558-1bfae74c9ea0","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-39057eed"}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"051c6d13-3acd-4bac-93eb-ff631782e196","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"namespace":"ai-tenant-e2e-mt-ea2ff266","name":"e2e-shared-int-sub-39057eed","reconcileID":"93d2225b-26c7-407c-9558-1bfae74c9ea0","MaaSSubscription":{"name":"e2e-shared-int-sub-39057eed","namespace":"ai-tenant-e2e-mt-ea2ff266"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"571d8e19-960e-4bd0-ac9b-264ad00ce5ea","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:17:58Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ddc6398e-288a-44bc-9a1a-cf689a1d61ab","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:18:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"39ef8fd8-2936-4bf3-a835-0c06ef139e26","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:18:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"e61d1d7d-74f4-44ce-8b55-6c85884c4d1e","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:18:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"6d8d9c42-edf2-4ec0-be53-0370025bc08a","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T20:18:11Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:18:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"eb3e756b-b00b-46a5-aca5-2461c7edd8b1","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:18:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"eb3e756b-b00b-46a5-aca5-2461c7edd8b1","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:18:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a1d49a58-9dd7-4a23-a88e-a870f76eaa8f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:18:15Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-8251a465/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T20:18:15Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"namespace":"ai-tenant-e2e-mt-8251a465","name":"e2e-policy-8251a465","reconcileID":"500cce03-ccba-49d7-8e15-f777fffbae91","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T20:18:15Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"namespace":"ai-tenant-e2e-mt-8251a465","name":"e2e-policy-8251a465","reconcileID":"500cce03-ccba-49d7-8e15-f777fffbae91"}
{"level":"info","ts":"2026-07-13T20:18:15Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"namespace":"ai-tenant-e2e-mt-8251a465","name":"e2e-policy-8251a465","reconcileID":"500cce03-ccba-49d7-8e15-f777fffbae91","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T20:18:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e16434ea-8de3-4cb4-b1f7-53aec3387e6a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T20:18:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:18:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:18:36Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"namespace":"ai-tenant-e2e-mt-8251a465","name":"e2e-policy-8251a465","reconcileID":"a77cac19-3bbb-40e7-b42c-9396bf83cd98","MaaSAuthPolicy":{"name":"e2e-policy-8251a465","namespace":"ai-tenant-e2e-mt-8251a465"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T20:18:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:18:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"5bd12ae3-2596-4324-9acd-e83828b7d089","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:18:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"5bd12ae3-2596-4324-9acd-e83828b7d089","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:18:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:18:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"cf407f7a-7fd1-45bb-b17a-5416f1a285c5","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:18:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"cf407f7a-7fd1-45bb-b17a-5416f1a285c5","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:18:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:18:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"b5533372-ba9d-4a17-a07d-3d12c2998275","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:18:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"b5533372-ba9d-4a17-a07d-3d12c2998275","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:18:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:18:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:18:56Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:18:56Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"371b8570-e5dc-4e08-81b9-789a594b824d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:18:56Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"371b8570-e5dc-4e08-81b9-789a594b824d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:19:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:19:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:19:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:19:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"9add6150-bc46-44c0-8871-0312f2c154ef","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:19:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"9add6150-bc46-44c0-8871-0312f2c154ef","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:19:41Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:19:41Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c98e893b-c7cd-4624-b692-f009ad28d3f8","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:19:41Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c98e893b-c7cd-4624-b692-f009ad28d3f8","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:19:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:19:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:20:15Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:20:15Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"d5c0d739-c533-457c-a343-ef82f91bf46f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:20:15Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"d5c0d739-c533-457c-a343-ef82f91bf46f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:20:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:20:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:20:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:20:26Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"48b8c073-5c93-4884-bc8b-e5be6f513eff","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:20:26Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"48b8c073-5c93-4884-bc8b-e5be6f513eff","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:20:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:20:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:21:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:21:00Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"c6b58d93-826c-471f-bbd8-757a9923372c","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:21:00Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"c6b58d93-826c-471f-bbd8-757a9923372c","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:21:11Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:21:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ea443942-a429-4e2f-8397-a798e4f97176","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:21:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ea443942-a429-4e2f-8397-a798e4f97176","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:21:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:21:24Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:21:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:21:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"4cb5b603-a38b-49f0-9fb9-650f22c37072","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:21:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"4cb5b603-a38b-49f0-9fb9-650f22c37072","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"error","ts":"2026-07-13T20:21:54Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"72c28a2a-d17e-49d7-ab0f-61d1651759a2","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:21:54Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"72c28a2a-d17e-49d7-ab0f-61d1651759a2","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","namespace":"ai-tenant-e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","count":0}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-917d1b/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-917d1b","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-917d1b","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-917d1b"},"tenantIdentifier":"e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","tenant":"e2e-inf-a-917d1b","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T20:21:54Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-917d1b","deploymentName":"maas-api-e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","gvk":"Deployment","name":"maas-api-e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","gvk":"Service","name":"maas-api-e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:54Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-917d1b"},"namespace":"ai-tenant-e2e-inf-a-917d1b","name":"default-tenant","reconcileID":"14564b09-82b1-44da-89e0-53419b18d41e","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-a-917d1b"}
{"level":"info","ts":"2026-07-13T20:21:56Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T20:21:56Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"71a5dc41-c086-4fab-aa79-5a08cfa781f9","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T20:21:56Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"71a5dc41-c086-4fab-aa79-5a08cfa781f9","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}