{"level":"error","ts":"2026-07-08T19:43:29Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"50889d34-fa2a-4245-a2c4-1a1f8f830cff","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:29Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-ec12e5f8/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-ec12e5f8","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-ec12e5f8","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-ec12e5f8"},"tenantIdentifier":"e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-ec12e5f8/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-ec12e5f8","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-ec12e5f8","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-ec12e5f8"},"tenantIdentifier":"e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","namespace":"ai-tenant-e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","count":0}
{"level":"info","ts":"2026-07-08T19:43:29Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-ait-ec12e5f8/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-ait-ec12e5f8","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-ait-ec12e5f8","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-ait-ec12e5f8"},"tenantIdentifier":"e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","tenant":"e2e-ait-ec12e5f8","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-08T19:43:29Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-ait-ec12e5f8","deploymentName":"maas-api-e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","gvk":"Deployment","name":"maas-api-e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","gvk":"Service","name":"maas-api-e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","gvk":"HTTPRoute","name":"maas-api-route-e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-ec12e5f8"},"namespace":"ai-tenant-e2e-ait-ec12e5f8","name":"default-tenant","reconcileID":"82e0a518-3a4e-4cd6-acc4-fea23297cd12","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-ait-ec12e5f8"}
{"level":"info","ts":"2026-07-08T19:43:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:43:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:43:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"error","ts":"2026-07-08T19:43:37Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"97405f99-f175-4660-8ed4-a0e21b4c90b1","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:43:37Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"97405f99-f175-4660-8ed4-a0e21b4c90b1","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"error","ts":"2026-07-08T19:43:37Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"6bbe7e71-388b-4f55-843b-667370fcd973","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:43:37Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"6bbe7e71-388b-4f55-843b-667370fcd973","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:37Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"6bbe7e71-388b-4f55-843b-667370fcd973","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","tenantID":"e2e-derive-e2758a2d","subscriptionNamespace":"ai-tenant-e2e-derive-e2758a2d","gatewayName":"e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:37Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-e2758a2d","deploymentName":"maas-api-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:37Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-e2758a2d","deploymentName":"maas-api-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:37Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-e2758a2d","deploymentName":"maas-api-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:38Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-e2758a2d","deploymentName":"maas-api-e2e-derive-e2758a2d"}
{"level":"error","ts":"2026-07-08T19:43:38Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"6bbe7e71-388b-4f55-843b-667370fcd973","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"error","ts":"2026-07-08T19:43:38Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"7eeac302-495c-4bfc-9faf-83c8ed9e0429","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:43:38Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"7eeac302-495c-4bfc-9faf-83c8ed9e0429","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"error","ts":"2026-07-08T19:43:38Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"fbc5774c-39b1-4429-826b-5d72832811f4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:43:38Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"fbc5774c-39b1-4429-826b-5d72832811f4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:40Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"60a52aab-c0b4-442c-813e-28814eef0cf6","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:43:40Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c942a4b8-3127-4847-a307-b0b91925c1d6","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:43:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"error","ts":"2026-07-08T19:43:41Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"f411b854-824b-4b3a-b648-ab56f61d4cd0","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:43:41Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"f411b854-824b-4b3a-b648-ab56f61d4cd0","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:41Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","namespace":"ai-tenant-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:41Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","count":0}
{"level":"info","ts":"2026-07-08T19:43:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-derive-e2758a2d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-derive-e2758a2d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-derive-e2758a2d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-derive-e2758a2d"},"tenantIdentifier":"e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:41Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","tenant":"e2e-derive-e2758a2d","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-08T19:43:41Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-derive-e2758a2d","deploymentName":"maas-api-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:41Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","gvk":"Deployment","name":"maas-api-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:41Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","gvk":"Service","name":"maas-api-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:42Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","gvk":"HTTPRoute","name":"maas-api-route-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:42Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-e2758a2d"},"namespace":"ai-tenant-e2e-derive-e2758a2d","name":"default-tenant","reconcileID":"59f47f24-4f78-4137-9dc1-f47b44905912","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-derive-e2758a2d"}
{"level":"info","ts":"2026-07-08T19:43:46Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:43:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d12b8ba8-0434-4632-a294-f3b8eaba5a36","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:43:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d12b8ba8-0434-4632-a294-f3b8eaba5a36","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:43:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c254e129-3d5e-4a53-a26a-c8940ecdbe40","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5fbee4fe-75cc-4e7e-bd60-0eaa46ac235a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-0e2690c7/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-policy-0e2690c7","reconcileID":"a6edfd98-8079-48ea-80e8-f8961617e16b","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-policy-0e2690c7","reconcileID":"a6edfd98-8079-48ea-80e8-f8961617e16b"}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-policy-0e2690c7","reconcileID":"a6edfd98-8079-48ea-80e8-f8961617e16b","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"54af77d2-6199-4e7c-b397-51dea2165555","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1b7bc4ed-d581-4b20-b8a6-cd268a21f260","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"865622ce-61fa-4283-bb75-36f563e409b7","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-sub-0e2690c7","reconcileID":"31418d9a-baa5-460b-8f54-a30192f914ae","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-0e2690c7/e2e-sub-0e2690c7","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9e2f7790-0f73-4a73-9b8a-a2a3bd15375e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"3d238105-133b-4154-ac64-213e18170202","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-sub-0e2690c7","reconcileID":"e91503e5-2c93-4b84-ad72-f91141b09551","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-policy-0e2690c7","reconcileID":"0132c674-06f3-4eae-b2c6-34bc6846704b","MaaSAuthPolicy":{"name":"e2e-policy-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dce7fcae-0180-4c9f-9eb6-b85ae65ba606","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"33c45d8d-b575-4056-a81e-e4042e6e4214","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:50Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-sub-0e2690c7","reconcileID":"e9fee1a2-4fb4-424b-b2d1-0dca738fa29a","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-sub-0e2690c7"}
{"level":"info","ts":"2026-07-08T19:43:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"73afa2b1-bd1f-4548-a97e-aa147c50ca17","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:50Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"namespace":"ai-tenant-e2e-mt-0e2690c7","name":"e2e-sub-0e2690c7","reconcileID":"e9fee1a2-4fb4-424b-b2d1-0dca738fa29a","MaaSSubscription":{"name":"e2e-sub-0e2690c7","namespace":"ai-tenant-e2e-mt-0e2690c7"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:43:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8d73781b-3515-46ca-b61d-71d5f755c5c1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"0203b4f1-2914-4923-875e-dd4dbd6e0ad6","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:43:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"6582b8e9-1325-48ee-b6bf-6254cae93048","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:43:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"30ece399-3441-4a3e-8ebf-2789e9072e08","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:43:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fe04d0d4-0b97-4cdc-a67b-a12de9af38f1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"87adc826-65d3-4668-81cd-0a7d9bf33948","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:56Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-c0ad5ad5/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:43:56Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"namespace":"ai-tenant-e2e-mt-c0ad5ad5","name":"e2e-policy-c0ad5ad5","reconcileID":"782a0f5e-9101-4d8f-80e5-c9ff2183fb8a","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:43:56Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"namespace":"ai-tenant-e2e-mt-c0ad5ad5","name":"e2e-policy-c0ad5ad5","reconcileID":"782a0f5e-9101-4d8f-80e5-c9ff2183fb8a"}
{"level":"info","ts":"2026-07-08T19:43:56Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"namespace":"ai-tenant-e2e-mt-c0ad5ad5","name":"e2e-policy-c0ad5ad5","reconcileID":"782a0f5e-9101-4d8f-80e5-c9ff2183fb8a","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:43:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d4559f4e-1230-4245-b163-7010e14243bf","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:43:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:43:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:44:22Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"namespace":"ai-tenant-e2e-mt-c0ad5ad5","name":"e2e-policy-c0ad5ad5","reconcileID":"b7a0c496-bd8c-4aee-a570-737d01ca10ea","MaaSAuthPolicy":{"name":"e2e-policy-c0ad5ad5","namespace":"ai-tenant-e2e-mt-c0ad5ad5"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:44:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:44:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:44:31Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:44:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7008891f-07ff-4ed1-8c59-6f0bb44bdbb1","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:44:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7008891f-07ff-4ed1-8c59-6f0bb44bdbb1","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:44:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:44:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:45:00Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-d848d99f/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:45:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"458c8a7d-a61f-40ef-ac7c-fbf5cba5c3b0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:00Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"namespace":"ai-tenant-e2e-mt-d848d99f","name":"e2e-policy-d848d99f","reconcileID":"784d9a2e-1da5-489b-a77d-53108244f707","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:45:00Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"namespace":"ai-tenant-e2e-mt-d848d99f","name":"e2e-policy-d848d99f","reconcileID":"784d9a2e-1da5-489b-a77d-53108244f707"}
{"level":"info","ts":"2026-07-08T19:45:00Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"namespace":"ai-tenant-e2e-mt-d848d99f","name":"e2e-policy-d848d99f","reconcileID":"784d9a2e-1da5-489b-a77d-53108244f707","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:45:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d005ae74-07ec-4d8b-a203-c71834d84728","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:01Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"namespace":"ai-tenant-e2e-mt-d848d99f","name":"e2e-policy-d848d99f","reconcileID":"d7016391-54cb-4f42-9f83-4cf172369695","MaaSAuthPolicy":{"name":"e2e-policy-d848d99f","namespace":"ai-tenant-e2e-mt-d848d99f"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:45:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c5615011-0d20-4c08-908b-1ee207cb14a2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2d109028-0198-487e-bdd2-baf1ba79d55a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"59229935-b91f-4ac8-b2bb-5b5dd4c7aad2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"824d4014-e630-4429-a8ed-34b00e44106d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-288dac51/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-policy-288dac51","reconcileID":"3e6450bf-0862-4c85-ad7b-b086fe40be5a","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-policy-288dac51","reconcileID":"3e6450bf-0862-4c85-ad7b-b086fe40be5a"}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-policy-288dac51","reconcileID":"3e6450bf-0862-4c85-ad7b-b086fe40be5a","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c337793d-dba4-4d25-8449-abd39d8819ff","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2397253e-394f-43dc-a734-85933ed9c614","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"de5e3e6d-693c-4a21-93f2-776c26dc188d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-sub-288dac51","reconcileID":"216bc7f7-6623-49e7-a0ba-e8b7f9ca3829","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-288dac51/e2e-shared-sub-288dac51","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"afeadd85-171b-441f-929a-a325afc675b0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"8652f5e3-df57-4e84-9ec7-c278906548b9","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-sub-288dac51","reconcileID":"415d9aea-ce79-4a49-a6c1-1ad1b641570f","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0b71c166-cd16-4bdf-8471-cf530a16bec1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2f782839-df20-4c2d-a396-e9753c027097","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:08Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-5a0c6280/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:45:08Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-policy-288dac51","reconcileID":"da3221bb-c177-4c77-8cbd-ce252e8324d5","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-policy-288dac51","reconcileID":"da3221bb-c177-4c77-8cbd-ce252e8324d5"}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-policy-288dac51","reconcileID":"da3221bb-c177-4c77-8cbd-ce252e8324d5","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"deca5d07-e4c5-4ba2-a43b-3dd021ae7a2c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"81256d26-89d9-4119-ab2b-80e83c81cfaf","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9a0815d3-456d-47c7-a266-2f87ddbe676f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a114c1b0-1be5-47b0-bd31-36f3489156d5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-sub-288dac51","reconcileID":"3d9dfa74-066b-47c5-bdda-06f12c8ac2fc","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-288dac51/e2e-shared-sub-288dac51","ai-tenant-e2e-mt-5a0c6280/e2e-shared-sub-288dac51","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"324aba1b-7e77-4e45-a1be-56eadff7a58f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"f7536b01-1bae-4b31-8c27-cf6dbd7aa5aa","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-sub-288dac51","reconcileID":"98df1252-9f1a-4a7f-94a3-acb8dff36d60","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-08T19:45:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-sub-288dac51","reconcileID":"ea5e7cdd-f1fc-487c-9b03-f94fb7651530","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-policy-288dac51","reconcileID":"4e561575-ed08-4cc7-9d53-e9d97751fe8e","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5b87a5c5-f261-4efa-897a-b82f3941e5d5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7bf3f7c5-3d82-4df0-b27e-d1e7f5c9a474","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-sub-288dac51","reconcileID":"7f8414f5-ff83-4c54-9101-eaaad8625a18","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-sub-288dac51"}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"452dd922-3316-4988-8ed9-c599b11d6be0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"namespace":"ai-tenant-e2e-mt-288dac51","name":"e2e-shared-sub-288dac51","reconcileID":"7f8414f5-ff83-4c54-9101-eaaad8625a18","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-288dac51"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-5a0c6280/e2e-shared-sub-288dac51","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3c8f8998-e42a-436a-9a1c-88ea283b05ae","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4fc9ef69-bc52-419f-9de1-74131a18aea9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-sub-288dac51","reconcileID":"ef41cb76-3df6-40e7-a84e-1f0fadbb97de","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:45:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"126f76ff-d454-4657-a6c5-4ddd53f8ca42","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:45:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"2126d1f0-5c9c-4027-9c8c-a0c6838bc658","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:45:16Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"34b5a000-2047-4968-b051-968b2e6dfb05","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:45:16Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"34b5a000-2047-4968-b051-968b2e6dfb05","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:16Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-policy-288dac51","reconcileID":"684c7259-cee5-4a01-a1d3-45cf31060b87","MaaSAuthPolicy":{"name":"e2e-shared-policy-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:45:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"880d2354-6f05-4c45-af65-0a1f431f4902","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b7604cd3-ba1f-4cfd-9b43-a8a291d38abd","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"6e7adefc-849c-47fa-a725-9403243bf1b1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:45:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-sub-288dac51","reconcileID":"80f1622a-7707-45ca-8ed3-3cda1ee6a623","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:45:17Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-sub-288dac51","reconcileID":"3b97ccdc-9e9c-4cfb-81aa-42b82aca5fe3","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-sub-288dac51"}
{"level":"info","ts":"2026-07-08T19:45:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d46233af-f249-43d5-bff9-25e5d7eaac71","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:17Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"namespace":"ai-tenant-e2e-mt-5a0c6280","name":"e2e-shared-sub-288dac51","reconcileID":"3b97ccdc-9e9c-4cfb-81aa-42b82aca5fe3","MaaSSubscription":{"name":"e2e-shared-sub-288dac51","namespace":"ai-tenant-e2e-mt-5a0c6280"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:45:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6ad37739-352c-411b-89e3-9fdfef51ba4d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5a4bd88c-9f54-4c5a-b9e5-884704c16657","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:20Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d26a7694-0a4d-4a3f-989c-efb3e515b311","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"d76da450-ff0b-4cec-b663-acba974b446b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"23e3b3bc-79b8-426a-b80d-0c483ceeccf8","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:26Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"error","ts":"2026-07-08T19:45:26Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"a59cb3a8-bd94-45ee-9267-c197b484c0eb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:45:26Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"a59cb3a8-bd94-45ee-9267-c197b484c0eb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:26Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"error","ts":"2026-07-08T19:45:26Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"d58b25fb-7683-44ec-9133-aac068448ba6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:45:26Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"d58b25fb-7683-44ec-9133-aac068448ba6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:26Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:26Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"d58b25fb-7683-44ec-9133-aac068448ba6","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","tenantID":"e2e-mt-d97e56e9","subscriptionNamespace":"ai-tenant-e2e-mt-d97e56e9","gatewayName":"e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-d97e56e9","deploymentName":"maas-api-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-d97e56e9","deploymentName":"maas-api-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-d97e56e9","deploymentName":"maas-api-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:27Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:27Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-d97e56e9","deploymentName":"maas-api-e2e-mt-d97e56e9"}
{"level":"error","ts":"2026-07-08T19:45:27Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"d58b25fb-7683-44ec-9133-aac068448ba6","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:27Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"error","ts":"2026-07-08T19:45:27Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"b2890129-3ab7-468a-9da8-234fea2c9331","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:45:27Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"b2890129-3ab7-468a-9da8-234fea2c9331","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:27Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"error","ts":"2026-07-08T19:45:27Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"f960aed1-1385-41b3-9350-e6f94bfa0f96","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:45:27Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"f960aed1-1385-41b3-9350-e6f94bfa0f96","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:45:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:45:32Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a3d0d398-091c-4fe9-ab24-82fd79a0aa3b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:32Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"25bfd08c-5767-4296-8ce2-b21c059313cf","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a100a00d-da41-468a-aec3-4d047bf2e57f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"2293782a-a583-4809-a48b-e35a085a0119","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"error","ts":"2026-07-08T19:45:38Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"12ed2310-5008-4f8d-9483-2185b1a577fe","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:45:38Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"12ed2310-5008-4f8d-9483-2185b1a577fe","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:45:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","namespace":"ai-tenant-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","count":0}
{"level":"info","ts":"2026-07-08T19:45:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-d97e56e9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-d97e56e9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-d97e56e9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-d97e56e9"},"tenantIdentifier":"e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","tenant":"e2e-mt-d97e56e9","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-08T19:45:38Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-d97e56e9","deploymentName":"maas-api-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","gvk":"Deployment","name":"maas-api-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","gvk":"Service","name":"maas-api-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:38Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d97e56e9"},"namespace":"ai-tenant-e2e-mt-d97e56e9","name":"default-tenant","reconcileID":"77fc6827-9dcb-4f56-b6c9-a6ba44b18432","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-d97e56e9"}
{"level":"info","ts":"2026-07-08T19:45:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"cdc9fdf5-18e6-4bfd-b700-0250d323dc3f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"eed11d40-139a-47cd-b398-beb05413d807","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"cd52f860-8965-4748-a212-b8e3f9ab7a01","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:48Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"435e3902-8b1d-4f38-a635-c892190227a2","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"49ce7af5-717c-4e20-9ff3-be4d0bf65f6e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:57Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"96828ac6-286f-40f9-b604-c0a62023a3ca","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-1609e22d","reconcileID":"59b6a883-bab9-4c66-8142-575e050e1c50","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-1609e22d","reconcileID":"59b6a883-bab9-4c66-8142-575e050e1c50"}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-1609e22d","reconcileID":"59b6a883-bab9-4c66-8142-575e050e1c50","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b6927584-7143-4b56-9167-b9e4ab730490","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-1609e22d","reconcileID":"a43486f7-40e4-4dca-87ca-753920d90336","MaaSAuthPolicy":{"name":"e2e-default-ns-1609e22d","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"af8fe256-cfc6-4753-8048-9ed6fe01c7f3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fc49e6b9-dc36-4391-bbaf-0636828b2db0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9a099180-0323-45f4-a571-9e9a87c5dab4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5eb21a48-e376-466e-ab92-948befaa887d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:58Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"6d16bfb7-4af4-4c1b-9e15-897aec2b9d95","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"6d16bfb7-4af4-4c1b-9e15-897aec2b9d95"}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"6d16bfb7-4af4-4c1b-9e15-897aec2b9d95","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"34343b52-ddab-40f5-ac27-7c7eaff3216c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:58Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"d5f409e5-a10d-4af7-ad74-123f4bf2341c","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"d5f409e5-a10d-4af7-ad74-123f4bf2341c"}
{"level":"info","ts":"2026-07-08T19:45:58Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"d5f409e5-a10d-4af7-ad74-123f4bf2341c","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:45:59Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3a963dde","reconcileID":"7fa178b6-ad30-43d9-bab7-00e01dd24b04","MaaSAuthPolicy":{"name":"e2e-gw-auth-3a963dde","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:45:59Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"71b54627-79fa-40f3-88c8-e0ccbcdc39e3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:59Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"29cdcde0-e4ba-4ba2-8e9b-9864f0ce4648","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:45:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:45:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:46:01Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:46:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"31494d6c-9530-48cf-a1aa-d4db0239251f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"31494d6c-9530-48cf-a1aa-d4db0239251f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:07Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:46:07Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"1e7d3f8c-96de-444c-bd46-cc7176f06c91","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:07Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"1e7d3f8c-96de-444c-bd46-cc7176f06c91","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:07Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:46:08Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"4e6ec0ce-81f2-4e23-b76d-8770fae38d62","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:08Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"4e6ec0ce-81f2-4e23-b76d-8770fae38d62","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:08Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:46:08Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"a8b597df-d4d5-4dce-b5b7-026e62ffb5c3","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:08Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"a8b597df-d4d5-4dce-b5b7-026e62ffb5c3","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:08Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:46:08Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"c511f339-8dd9-4061-a405-5b6e864057e6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:08Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"c511f339-8dd9-4061-a405-5b6e864057e6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:46:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:46:46Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:46:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"aaeece06-20da-4d08-8b5c-80046c7bc72c","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"aaeece06-20da-4d08-8b5c-80046c7bc72c","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:46:52Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"2b456f94-2678-44b9-bebe-2593f4aa43c9","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:46:52Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"2b456f94-2678-44b9-bebe-2593f4aa43c9","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:46:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:46:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:47:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:47:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:47:31Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:47:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"16f8ffb0-32e3-4f69-a7b4-671136ce3dd4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:47:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"16f8ffb0-32e3-4f69-a7b4-671136ce3dd4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:47:37Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:47:37Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"a23f8c11-ba72-45e5-84d0-bf280e5e075e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:47:37Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"a23f8c11-ba72-45e5-84d0-bf280e5e075e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:47:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:47:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:48:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:48:16Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"a0151303-97e4-450e-a178-67e8d14634cc","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:48:16Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"a0151303-97e4-450e-a178-67e8d14634cc","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:48:22Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:48:22Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"4d80dd48-0720-40f2-84b3-9d14539b2a3e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:48:22Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"4d80dd48-0720-40f2-84b3-9d14539b2a3e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:48:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:48:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:48:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:48:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:49:01Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:49:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bd8b6e39-9b4a-434a-aee7-b5ab57eed379","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:49:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bd8b6e39-9b4a-434a-aee7-b5ab57eed379","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:49:07Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:49:07Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"d61bf750-79eb-4725-91f2-a7176777fa41","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:49:07Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"d61bf750-79eb-4725-91f2-a7176777fa41","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:49:17Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"error","ts":"2026-07-08T19:49:17Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"26491f3b-7824-4958-a67f-d7f403c02557","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:49:17Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"26491f3b-7824-4958-a67f-d7f403c02557","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:49:17Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","namespace":"ai-tenant-e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","count":0}
{"level":"info","ts":"2026-07-08T19:49:17Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-0dad1a0a/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-0dad1a0a","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-0dad1a0a","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-0dad1a0a"},"tenantIdentifier":"e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","tenant":"e2e-mt-0dad1a0a","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-08T19:49:17Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-0dad1a0a","deploymentName":"maas-api-e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","gvk":"Deployment","name":"maas-api-e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","gvk":"Service","name":"maas-api-e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:17Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0dad1a0a"},"namespace":"ai-tenant-e2e-mt-0dad1a0a","name":"default-tenant","reconcileID":"b15d4b72-64a3-4ae5-a73c-c4e9bd6efe5c","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-0dad1a0a"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8c432182-b4e3-4c58-a516-253a75d7be35","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"619f43ba-1c41-4b19-9063-40ffa2560aa4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-8bb79f","reconcileID":"71cf4e52-911a-4cd8-8a44-26b734ad9856","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-8bb79f","reconcileID":"71cf4e52-911a-4cd8-8a44-26b734ad9856"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-8bb79f","reconcileID":"71cf4e52-911a-4cd8-8a44-26b734ad9856","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d183b533-1959-4ca2-a377-79275fcddbda","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dce2e033-fe7a-4c7b-aece-e7f70029c42b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c72b22b1-fc1f-4209-8bb2-f77429842c91","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-08T19:49:24Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-8bb79f","reconcileID":"ae7cd74e-c776-4090-9381-e5b535de1b1d","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-default-int-sub-8bb79f\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fa2a2bea-fa5b-481f-afc7-67212ec89b42","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-8bb79f","reconcileID":"ec0c3ca8-f15f-4ed8-91fc-ed17735d9263","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-default-int-sub-8bb79f","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d3aa72fc-21a3-40c1-8a18-883d3b63e424","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"e0f2e8db-2220-45a9-9028-38ac935e12cc","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-8bb79f","reconcileID":"2d112c78-af68-48e4-b45a-c9ef442952e1","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5152f89f-3fb5-4510-be2e-cef92932d21c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-8bb79f","reconcileID":"311969a2-4f63-4b03-bc93-1f36215327cb","MaaSAuthPolicy":{"name":"e2e-default-int-auth-8bb79f","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d0ca9e89-2f12-458a-970e-ef8320904e57","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-8bb79f","reconcileID":"8028124f-fcd9-47e1-815c-6121d25c14f1","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-default-int-sub-8bb79f"}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"87b7758f-d913-485a-a058-aed465ad95a8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-8bb79f","reconcileID":"8028124f-fcd9-47e1-815c-6121d25c14f1","MaaSSubscription":{"name":"e2e-default-int-sub-8bb79f","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c2a3fd2d-7090-4178-aa1f-ee1bd7f7106f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"e0fbbcf6-d845-4ab6-846a-185133fd1d64","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c393104c-ee06-4a98-a8ce-068023271d06","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4bdb1b7f-f39c-4bc8-b94d-ba6cadb404d1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:25Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-308c485c/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-policy-308c485c","reconcileID":"7a72a1c9-259c-490f-b220-118099d4c2f2","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-policy-308c485c","reconcileID":"7a72a1c9-259c-490f-b220-118099d4c2f2"}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-policy-308c485c","reconcileID":"7a72a1c9-259c-490f-b220-118099d4c2f2","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b776081a-9d78-45bd-9ed4-06169ef97cde","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"14ea0b92-0684-4584-8ba2-823edeb0f8ab","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a58a86cd-a940-44f3-b76f-ba8ddd984bfd","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-08T19:49:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-sub-308c485c","reconcileID":"ff77f6c3-f6ea-4774-a7df-66de25bf7fe3","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-308c485c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6dd847eb-8754-45f4-91f4-deeb92909790","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-sub-308c485c","reconcileID":"01a99641-e812-4e07-9036-43890bcd8e4e","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-308c485c/e2e-shared-int-sub-308c485c","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e29425e6-0db2-46e1-99d3-137cbc49f9c1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ecba9ebf-27dd-4174-b904-7354db72fd4b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:49:25Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-sub-308c485c","reconcileID":"4623ff13-d624-4714-aabd-47d0f74cd253","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3eb49fed-4f93-430f-8a24-e419f9adc44f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c4a5e58c-9ef5-4a63-9ac1-2cd9f04f01fb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:26Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-b3f6fe90/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-policy-308c485c","reconcileID":"af645704-7d58-4139-90bb-b7c303b19982","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-policy-308c485c","reconcileID":"af645704-7d58-4139-90bb-b7c303b19982"}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-policy-308c485c","reconcileID":"af645704-7d58-4139-90bb-b7c303b19982","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"17296110-5f05-445a-b771-30377ee7bf8d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7ff1ee4b-81bb-47fc-a7dc-2a03546c5414","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c64a246d-c26a-4051-9323-683bc88620ee","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-sub-308c485c","reconcileID":"e6cddfac-ad45-4b29-b1f8-679cbc521617","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-308c485c/e2e-shared-int-sub-308c485c","ai-tenant-e2e-mt-b3f6fe90/e2e-shared-int-sub-308c485c","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"05a077be-2418-4295-aff4-827eece85aa9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"f78b3a81-cda7-4ece-85e5-035ffc4f868e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-sub-308c485c","reconcileID":"cb4d1be7-2712-4357-888a-ebdf67fe596a","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-08T19:49:26Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-sub-308c485c","reconcileID":"fde1e785-d1db-4c45-8670-16c4805814de","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-policy-308c485c","reconcileID":"25f83e89-9d59-4420-b49a-4d8a151962c6","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a1f60df0-8e34-4b82-91ac-037adf100b88","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b8dd4a86-a1c5-424d-a2ec-a052fe0ee40e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-sub-308c485c","reconcileID":"908fbd8b-dfb4-4973-a830-5819b8099e14","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-308c485c"}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6cc789d3-fede-4687-9e3c-50803667fb80","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"namespace":"ai-tenant-e2e-mt-308c485c","name":"e2e-shared-int-sub-308c485c","reconcileID":"908fbd8b-dfb4-4973-a830-5819b8099e14","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-308c485c"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-b3f6fe90/e2e-shared-int-sub-308c485c","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2e72ab90-801b-4372-9cef-3e210c987f3c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"23e5195a-d2ba-48f5-ac55-c322a4b542a6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"34b2b09d-4cea-4c28-9415-25da20f0891c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:49:27Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-sub-308c485c","reconcileID":"373af65d-f487-4c58-9792-7061bda405a5","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-08T19:49:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:49:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:49:30Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"574be9a4-1392-4169-b14e-f1b9f8d6c839","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:49:33Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-policy-308c485c","reconcileID":"89196d3e-2884-44d6-945a-a32e66e7301c","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:49:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8d528a29-1ee1-40ec-bdf4-68a41dcb63f3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"57f25b6b-654e-4a38-ba97-874363491949","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:34Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-sub-308c485c","reconcileID":"3027eba2-90e7-407a-a516-b263279eae5e","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-308c485c"}
{"level":"info","ts":"2026-07-08T19:49:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5850266c-cd3a-4c20-a105-0a10b418e0b4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:34Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"namespace":"ai-tenant-e2e-mt-b3f6fe90","name":"e2e-shared-int-sub-308c485c","reconcileID":"3027eba2-90e7-407a-a516-b263279eae5e","MaaSSubscription":{"name":"e2e-shared-int-sub-308c485c","namespace":"ai-tenant-e2e-mt-b3f6fe90"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-08T19:49:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8b0af6ee-9aac-4dd4-909d-52f754221040","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a481d80c-4e83-4752-8dab-a0f93f6937b4","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:49:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"42b37560-7da2-4f9f-a165-41381e4d76cf","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:49:44Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"ea8fb312-bdf8-491f-8fd7-b5580d3881f6","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:49:44Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"364ba6eb-197a-467b-be09-5a05a1f17a1c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:49:46Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:49:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0080f7fe-b196-443a-bdd9-d0216fc70713","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:49:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0080f7fe-b196-443a-bdd9-d0216fc70713","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:49:50Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-11e7b674/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-08T19:49:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"34a2e98c-62c8-4b15-a0dd-637a82ed27f9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"namespace":"ai-tenant-e2e-mt-11e7b674","name":"e2e-policy-11e7b674","reconcileID":"08b20980-91c2-4be8-b88d-74792858a651","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-08T19:49:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"namespace":"ai-tenant-e2e-mt-11e7b674","name":"e2e-policy-11e7b674","reconcileID":"08b20980-91c2-4be8-b88d-74792858a651"}
{"level":"info","ts":"2026-07-08T19:49:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"namespace":"ai-tenant-e2e-mt-11e7b674","name":"e2e-policy-11e7b674","reconcileID":"08b20980-91c2-4be8-b88d-74792858a651","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-08T19:49:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"68bc03d4-c69a-419d-8627-205936428cdb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-08T19:49:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:49:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:50:11Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"namespace":"ai-tenant-e2e-mt-11e7b674","name":"e2e-policy-11e7b674","reconcileID":"760b75a5-0479-4147-9127-6e7d5203a48d","MaaSAuthPolicy":{"name":"e2e-policy-11e7b674","namespace":"ai-tenant-e2e-mt-11e7b674"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-08T19:50:22Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:50:22Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"ec8427f1-7e35-4774-9636-5bc1361d3558","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:22Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"ec8427f1-7e35-4774-9636-5bc1361d3558","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:22Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:50:22Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"18f3508c-4f43-4f1d-8091-36bd6087d270","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:22Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"18f3508c-4f43-4f1d-8091-36bd6087d270","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:22Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:50:22Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"18f3508c-4f43-4f1d-8091-36bd6087d270","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","tenantID":"e2e-inf-a-0df657","subscriptionNamespace":"ai-tenant-e2e-inf-a-0df657","gatewayName":"e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:50:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-0df657","deploymentName":"maas-api-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:50:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-0df657","deploymentName":"maas-api-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:50:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-0df657","deploymentName":"maas-api-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:50:23Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:50:23Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-0df657","deploymentName":"maas-api-e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:50:23Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"18f3508c-4f43-4f1d-8091-36bd6087d270","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:23Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:50:23Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"005f1df4-c14f-4ff6-bccd-f666dfad3baf","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:23Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"005f1df4-c14f-4ff6-bccd-f666dfad3baf","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:23Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:50:23Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"4dd86fe2-5af8-44f4-8df7-254663240c18","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:23Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"4dd86fe2-5af8-44f4-8df7-254663240c18","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:50:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:50:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:50:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"50d3149b-d449-4efe-8d3e-f1865f0ea5e8","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"50d3149b-d449-4efe-8d3e-f1865f0ea5e8","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:50:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"fa2cc4bc-507d-4606-ac28-39c25b7da891","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"fa2cc4bc-507d-4606-ac28-39c25b7da891","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:50:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"7d8a733c-f104-4380-8644-8293bc5c4106","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"7d8a733c-f104-4380-8644-8293bc5c4106","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:30Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"02f2f7f0-6a35-4fa6-b3ba-dc8e3a14749c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:50:30Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"439d5a3f-5c5a-416c-9c48-c67ca1be47a4","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:50:31Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:50:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"97b1e1a7-b1ad-46b8-b56b-02a02ea877fa","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:50:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"97b1e1a7-b1ad-46b8-b56b-02a02ea877fa","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:50:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"3ddeab95-2a27-4336-a05b-10adb3b0005b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:50:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0a6c30ba-e110-4bd6-9743-734f97f04ed4","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:50:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:50:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:51:08Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:51:08Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"d8665f9e-73e1-4317-8c00-1d200a68bbb7","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:51:08Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"d8665f9e-73e1-4317-8c00-1d200a68bbb7","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:51:15Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:51:15Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"af6d602e-7ee3-4180-a44f-e0a3198a4aa4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:51:15Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"af6d602e-7ee3-4180-a44f-e0a3198a4aa4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:51:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:51:16Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"b8685a66-a935-4930-a09f-2104ec0d7cfd","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:51:16Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"b8685a66-a935-4930-a09f-2104ec0d7cfd","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:51:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:51:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:51:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:51:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"2c78a484-2ec6-4f15-a884-6bf4db8057fb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:51:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"2c78a484-2ec6-4f15-a884-6bf4db8057fb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:51:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:51:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:52:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:52:00Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"ae7040ea-4ee8-46f5-98c4-608c5bea1c10","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:52:00Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"ae7040ea-4ee8-46f5-98c4-608c5bea1c10","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:52:01Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:52:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bd291266-5175-454d-916f-3007043313eb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:52:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bd291266-5175-454d-916f-3007043313eb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:52:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:52:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:52:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:52:38Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"2fb766f5-4a7d-4b2a-b6e5-a718e03dae7d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:52:38Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"2fb766f5-4a7d-4b2a-b6e5-a718e03dae7d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:52:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:52:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"318feea1-0be2-48f4-be56-b82bc1bafadd","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:52:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"318feea1-0be2-48f4-be56-b82bc1bafadd","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:52:46Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:52:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c779b663-4f16-404a-bbb2-19acb88d7f5e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:52:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c779b663-4f16-404a-bbb2-19acb88d7f5e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:52:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:52:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:53:23Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:53:23Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"5a8e539e-ab6b-4f2c-be9b-37ccf10a1e95","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:53:23Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"5a8e539e-ab6b-4f2c-be9b-37ccf10a1e95","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:53:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:53:29Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:53:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:53:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"efe90f32-4da7-4b3d-9545-d62d204ad2fc","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:53:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"efe90f32-4da7-4b3d-9545-d62d204ad2fc","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:53:31Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-08T19:53:31Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"17f77101-28ac-4c8c-b328-154ecdec1a4d","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:53:31Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"17f77101-28ac-4c8c-b328-154ecdec1a4d","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:53:39Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"error","ts":"2026-07-08T19:53:39Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"a1f19e53-3021-4644-a704-5b6d8dd88e2f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:53:39Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"a1f19e53-3021-4644-a704-5b6d8dd88e2f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:53:39Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","namespace":"ai-tenant-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","count":0}
{"level":"info","ts":"2026-07-08T19:53:39Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-0df657/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-0df657","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-0df657","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-0df657"},"tenantIdentifier":"e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","tenant":"e2e-inf-a-0df657","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-08T19:53:39Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-0df657","deploymentName":"maas-api-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","gvk":"Deployment","name":"maas-api-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","gvk":"Service","name":"maas-api-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:39Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0df657"},"namespace":"ai-tenant-e2e-inf-a-0df657","name":"default-tenant","reconcileID":"9076eb3f-cb30-4674-a74a-5be6b839d8d8","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-a-0df657"}
{"level":"info","ts":"2026-07-08T19:53:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"503cd85d-47b6-4373-8f09-83ae6e644fd0","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:53:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c0f5e133-52a0-4a32-b800-3520f55df983","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:53:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"error","ts":"2026-07-08T19:53:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"cec0fec8-a1c3-411c-888b-46a69a60563e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-08T19:53:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"cec0fec8-a1c3-411c-888b-46a69a60563e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-08T19:53:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","namespace":"ai-tenant-e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","count":0}
{"level":"info","ts":"2026-07-08T19:53:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-ed6cb9/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-ed6cb9","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-ed6cb9","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"tenantIdentifier":"e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","tenant":"e2e-inf-b-ed6cb9","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-08T19:53:46Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-ed6cb9","deploymentName":"maas-api-e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","gvk":"Deployment","name":"maas-api-e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","gvk":"Service","name":"maas-api-e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-ed6cb9"},"namespace":"ai-tenant-e2e-inf-b-ed6cb9","name":"default-tenant","reconcileID":"edfb8f62-b55b-4b21-a2ca-8d6f2aec18ab","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-b-ed6cb9"}
{"level":"info","ts":"2026-07-08T19:53:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"775e91cd-0f2a-42f8-9b1e-b3904c518ac9","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:53:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"b2723685-34eb-4624-816f-1f304aedc39b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-08T19:53:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-08T19:53:59Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}