{"level":"info","ts":"2026-07-09T03:10:51Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3f638745","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3f638745","reconcileID":"eab10e96-903f-4003-a2d8-02ee1d59e3f3"}
{"level":"info","ts":"2026-07-09T03:10:51Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3f638745","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3f638745","reconcileID":"eab10e96-903f-4003-a2d8-02ee1d59e3f3","MaaSAuthPolicy":{"name":"e2e-gw-auth-3f638745","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:10:52Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-3f638745","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-3f638745","reconcileID":"1c31e9ab-a493-48e0-be5d-47ac0d0cef52","MaaSAuthPolicy":{"name":"e2e-gw-auth-3f638745","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T03:10:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c912ff72-19ce-4222-9fd6-ca340a63ce13","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:10:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"47762d6a-c21f-49a2-bf18-bc151e76f933","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"error","ts":"2026-07-09T03:11:00Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"db0bebc8-67ef-42c4-bf65-f48d8e29e459","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:00Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"db0bebc8-67ef-42c4-bf65-f48d8e29e459","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"error","ts":"2026-07-09T03:11:00Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"1a824342-098e-4b66-a3e0-e3ea10150522","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:00Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"1a824342-098e-4b66-a3e0-e3ea10150522","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:00Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"1a824342-098e-4b66-a3e0-e3ea10150522","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","tenantID":"e2e-mt-28444007","subscriptionNamespace":"ai-tenant-e2e-mt-28444007","gatewayName":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:00Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-28444007","deploymentName":"maas-api-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:00Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-28444007","deploymentName":"maas-api-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:00Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-28444007","deploymentName":"maas-api-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:01Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-28444007","deploymentName":"maas-api-e2e-mt-28444007"}
{"level":"error","ts":"2026-07-09T03:11:01Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"1a824342-098e-4b66-a3e0-e3ea10150522","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"error","ts":"2026-07-09T03:11:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"ef06e65e-c4d7-4c99-a779-c1feca615d85","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"ef06e65e-c4d7-4c99-a779-c1feca615d85","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"error","ts":"2026-07-09T03:11:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"d68643be-0ca7-44a7-817a-05e063ab9609","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"d68643be-0ca7-44a7-817a-05e063ab9609","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:07Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"3cb2943c-728b-4f52-9df8-5f6db557044b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:08Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4a5f8f91-37c2-47c5-8b18-42b94d706cfb","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:11:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:11:10Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:11:10Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bf8de9e5-b11c-4fdd-b8b8-ca162fd35398","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:10Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bf8de9e5-b11c-4fdd-b8b8-ca162fd35398","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"40b8e5bd-775f-4586-a1f7-82077d602aa6","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"1da4250e-5372-41cb-886c-a4d3451b75b1","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"310befaa-c992-4937-87d3-0eb2e2e14b1c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"7505582a-ce76-4cbb-b7fb-e3731a853b29","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:11:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5af9be2e-a440-4c7d-97d1-490cd802a712","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"1833468a-e4c7-4597-b34d-f1e8f9abee75","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"35615d0c-c74c-4671-b102-1d03d1514b89","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:41Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"026d9f3d-c541-4e3a-be72-86c6ddbc9112","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-28444007","tenantID":"e2e-mt-28444007","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"026d9f3d-c541-4e3a-be72-86c6ddbc9112"}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"026d9f3d-c541-4e3a-be72-86c6ddbc9112","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"e2e-mt-28444007-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:11:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"af2589fc-02cd-4fe0-b5ce-fc68a1479f9e","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"6e456f7f-c9ae-4ff0-90e2-a64ac473bb71","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"32351ef5-0632-4e6f-95c9-d55f75a8f777","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"error","ts":"2026-07-09T03:11:42Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"b15b1441-0d4f-483a-8915-8b866a9f1f4e","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-sub-28444007\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:42Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"706b3462-2736-47e4-8bc9-c4fc3eed5f5b","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-28444007","tenantID":"e2e-mt-28444007","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-09T03:11:42Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"32351ef5-0632-4e6f-95c9-d55f75a8f777","name":"e2e-lifecycle-model-28444007","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"e2e-lifecycle-model-28444007\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"75f9a50c-f527-4621-8ddc-f4173d7f5641","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"706b3462-2736-47e4-8bc9-c4fc3eed5f5b"}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"706b3462-2736-47e4-8bc9-c4fc3eed5f5b","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"e2e-mt-28444007-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"e60a1c88-09b2-4ed1-a1bc-32ffdb743913","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","model":"e2e-lifecycle-model-28444007","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-mt-28444007/e2e-sub-28444007"]}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"d6afbb62-386b-4bf7-b8ca-cf64371a4ff0","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"9097fd43-fa2c-4c98-8651-d9a1cfcdcbe2","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a25b7e57-7282-48a5-b700-220d2f8d3aee","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"error","ts":"2026-07-09T03:11:46Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"bfc33d81-f4a5-459f-8f7d-f85e81d5a29b","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:46Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"bfc33d81-f4a5-459f-8f7d-f85e81d5a29b","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"2e66ba14-ce61-4002-b822-3cee4bf8d55f","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"fb300ba4-2a3d-4370-b528-521aec63944f","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"56cdde36-2ef9-4ebe-87ab-4a8d0935c16c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"eea51497-4032-4c1e-9573-17219db60793","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:46Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"c8cf7970-4b12-4ee3-a0d5-f02d8771aa56","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-28444007","tenantID":"e2e-mt-28444007","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"0d925f45-eca1-4ba5-9810-e4148f93fa7b","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"c8cf7970-4b12-4ee3-a0d5-f02d8771aa56"}
{"level":"info","ts":"2026-07-09T03:11:46Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"c8cf7970-4b12-4ee3-a0d5-f02d8771aa56","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"e2e-mt-28444007-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:11:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"e5399bed-ee9d-42fd-9c23-d6a443f5fcd4","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-28444007","tenantID":"e2e-mt-28444007","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-09T03:11:47Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"75bd3d6f-8b36-4914-9c9e-989b6e963d2a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:47Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"75bd3d6f-8b36-4914-9c9e-989b6e963d2a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"7ca967f0-bd78-49c9-8a33-b2f9f5f9196f","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","namespace":"ai-tenant-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Deleting MaaSAuthPolicy","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"e5399bed-ee9d-42fd-9c23-d6a443f5fcd4"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"e5399bed-ee9d-42fd-9c23-d6a443f5fcd4","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"e2e-mt-28444007-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","count":1}
{"level":"info","ts":"2026-07-09T03:11:47Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-28444007/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-28444007","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-28444007","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-28444007"},"tenantIdentifier":"e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","tenant":"e2e-mt-28444007","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-09T03:11:47Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-28444007","deploymentName":"maas-api-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"966dea3e-c375-44da-9dd5-9fc429e35820","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","namespace":"ai-tenant-e2e-mt-28444007","llmisvcName":"e2e-lifecycle-model-28444007","gateway":"openshift-ingress/e2e-mt-28444007","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","gvk":"Deployment","name":"maas-api-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"7304ba16-b533-49ae-b7a3-c106890f731b","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","gvk":"Service","name":"maas-api-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"default-tenant","reconcileID":"436d9aa7-027d-42dc-9af6-5ef361ac0f29","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-28444007"}
{"level":"info","ts":"2026-07-09T03:11:47Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-policy-28444007","reconcileID":"03a2e2a6-be7b-454b-96d1-99a9113e9cc6","MaaSAuthPolicy":{"name":"e2e-policy-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007"}
{"level":"info","ts":"2026-07-09T03:11:48Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"f4fe51d5-6ed3-4024-bb7b-1d3180b218db","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007","subscription":"e2e-sub-28444007"}
{"level":"info","ts":"2026-07-09T03:11:48Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"f4fe51d5-6ed3-4024-bb7b-1d3180b218db","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007"}
{"level":"info","ts":"2026-07-09T03:11:48Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-sub-28444007","reconcileID":"f4fe51d5-6ed3-4024-bb7b-1d3180b218db","MaaSSubscription":{"name":"e2e-sub-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"name":"maas-trlp-e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007","model":"ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5e2a188b-8a1a-418b-9953-71c60712deba","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5e2a188b-8a1a-418b-9953-71c60712deba","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5e2a188b-8a1a-418b-9953-71c60712deba","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5f67387c-9559-4e8b-9a37-d4c932e28679","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5f67387c-9559-4e8b-9a37-d4c932e28679","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5f67387c-9559-4e8b-9a37-d4c932e28679","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"719d74a5-fa53-447d-8182-d4a9352c8b86","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"719d74a5-fa53-447d-8182-d4a9352c8b86","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"719d74a5-fa53-447d-8182-d4a9352c8b86","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"53406eab-2ff2-42b4-ba6e-45ebc729c002","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"53406eab-2ff2-42b4-ba6e-45ebc729c002","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"53406eab-2ff2-42b4-ba6e-45ebc729c002","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"64e6abd7-d923-49cc-9de9-d4c04db6dfac","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"64e6abd7-d923-49cc-9de9-d4c04db6dfac","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"64e6abd7-d923-49cc-9de9-d4c04db6dfac","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5ab5b7e2-2081-403d-bd1b-f5528c1ac101","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5ab5b7e2-2081-403d-bd1b-f5528c1ac101","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"5ab5b7e2-2081-403d-bd1b-f5528c1ac101","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"49a26ffc-fd18-4ac4-93a0-3e39d67f7487","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"49a26ffc-fd18-4ac4-93a0-3e39d67f7487","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:53Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"49a26ffc-fd18-4ac4-93a0-3e39d67f7487","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"f0c292b8-0be3-4d11-aa3f-a412c1c60103","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"f0c292b8-0be3-4d11-aa3f-a412c1c60103","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"f0c292b8-0be3-4d11-aa3f-a412c1c60103","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"e77bd0b4-b28c-4e4b-ae86-e24d4a912ca4","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"e77bd0b4-b28c-4e4b-ae86-e24d4a912ca4","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"e77bd0b4-b28c-4e4b-ae86-e24d4a912ca4","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:55Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"385be2b8-8ac2-44a1-bdb3-fa6a4d1c38e4","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:55Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"385be2b8-8ac2-44a1-bdb3-fa6a4d1c38e4","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:55Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"385be2b8-8ac2-44a1-bdb3-fa6a4d1c38e4","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:11:55Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:11:55Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c63961fb-8a24-433f-9bd5-76c9afafa458","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:55Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"c63961fb-8a24-433f-9bd5-76c9afafa458","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:58Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"08071cef-6ba9-4cef-be07-c392a6c7e73d","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:58Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"08071cef-6ba9-4cef-be07-c392a6c7e73d","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:11:58Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"08071cef-6ba9-4cef-be07-c392a6c7e73d","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:12:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"8dede492-9fc5-492a-a595-862c4eae16b7","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:12:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c273b458-153f-4f24-8fd2-2938ee642cbb","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T03:12:03Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"2571a6bf-b728-42c6-99a9-cc57ba8f1d27","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:03Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"2571a6bf-b728-42c6-99a9-cc57ba8f1d27","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:03Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"2571a6bf-b728-42c6-99a9-cc57ba8f1d27","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:12:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:12:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T03:12:13Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"e7cdd047-652a-44a4-b476-6374e5013436","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:13Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"e7cdd047-652a-44a4-b476-6374e5013436","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:13Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"e7cdd047-652a-44a4-b476-6374e5013436","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:29Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"8660ade9-3b43-4843-af42-ebe87e421d9f","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"routeName":"e2e-lifecycle-model-28444007-kserve-route","routeNamespace":"ai-tenant-e2e-mt-28444007","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-28444007","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:29Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"8660ade9-3b43-4843-af42-ebe87e421d9f","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:29Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"},"namespace":"ai-tenant-e2e-mt-28444007","name":"e2e-lifecycle-model-28444007","reconcileID":"8660ade9-3b43-4843-af42-ebe87e421d9f","error":"HTTPRoute ai-tenant-e2e-mt-28444007/e2e-lifecycle-model-28444007-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-28444007). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:12:33Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"66fabb18-4413-4b7b-b536-848e5b986ec3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:12:33Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"e66e8f46-03ab-4970-bc60-7657b65b7b03","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2304662a-f1e2-498e-9e1d-9505cd882636","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4a766479-6025-4968-aa47-a032335cf698","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:36Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-458985","reconcileID":"f3797c19-b3e0-4a2b-a8d5-7325ddfadec2","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-458985","reconcileID":"f3797c19-b3e0-4a2b-a8d5-7325ddfadec2"}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-458985","reconcileID":"f3797c19-b3e0-4a2b-a8d5-7325ddfadec2","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d29bc1fb-fd47-4808-9fa9-777b5cb686a0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1c210fe1-c023-4ed6-98d5-47b85da2ef5c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"779d7aaa-faf1-42f3-9002-f628de456d13","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-458985","reconcileID":"c43d908d-de28-45c9-9ed4-12b1a842f9fa","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-default-int-sub-458985","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"524d9be7-7ae2-4c85-b4d3-a056c23717c0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d674af9f-c4b2-4ae1-9a4c-f4741097b4d1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-09T03:12:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-458985","reconcileID":"2ea045b2-d220-465f-8a39-cc355348cb0a","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-458985","reconcileID":"8bd62898-cbc6-4033-b33c-323207326b75","MaaSAuthPolicy":{"name":"e2e-default-int-auth-458985","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"310423c8-ac25-45a1-aca6-3f445e80eb2d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"63c40aed-d811-4d76-84f8-25964e40f88d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-458985","reconcileID":"d8fcb237-3e0e-40c1-b60a-50c6ac0d9f13","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-default-int-sub-458985"}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fe58c7df-292e-40a9-b7e1-2cae719dae3c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-458985","reconcileID":"d8fcb237-3e0e-40c1-b60a-50c6ac0d9f13","MaaSSubscription":{"name":"e2e-default-int-sub-458985","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4cf952db-a581-4bb0-ba75-17ee5f721a94","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d3acabc2-f2b3-4cc0-8f2c-06d8556c9235","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"94829bbe-2770-4a99-8216-b5a2b674bf5f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"654c44bc-5e70-487f-8933-948c6437c6d8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:37Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-8d991807/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-policy-8d991807","reconcileID":"250a964a-1d7f-4c10-9908-d31084819444","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-policy-8d991807","reconcileID":"250a964a-1d7f-4c10-9908-d31084819444"}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-policy-8d991807","reconcileID":"250a964a-1d7f-4c10-9908-d31084819444","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:12:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"01d53372-57f1-4639-af8c-138d2b7dac67","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3d84a527-aaf7-4760-aa08-03a8833fc612","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e710f08e-005b-44bc-a8f2-123aeacc17fc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T03:12:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-sub-8d991807","reconcileID":"b1663804-e11c-405d-b49c-5cd0f83a1c7f","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-8d991807\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"cf2cbbec-f0d0-40e8-a3a1-aa2d28d26258","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-sub-8d991807","reconcileID":"89923626-8aeb-4b5c-be23-a26ca8f3638f","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-8d991807/e2e-shared-int-sub-8d991807","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"52ad57e0-378d-40d1-bd45-5f4c0a0b562d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"775e57c2-6115-4a24-81fb-610edffb9574","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-sub-8d991807","reconcileID":"39d2f025-8d46-4ddc-b3bf-879d62cd4b02","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-09T03:12:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:12:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d607e636-7ea0-4c88-b403-ef40d2dab407","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d742af7b-e14b-499f-9753-fc6331c99b52","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-e9c8e886/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-policy-8d991807","reconcileID":"cb7cfd7c-a78b-4603-96cd-c850dbf4d778","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-policy-8d991807","reconcileID":"cb7cfd7c-a78b-4603-96cd-c850dbf4d778"}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-policy-8d991807","reconcileID":"cb7cfd7c-a78b-4603-96cd-c850dbf4d778","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"01ae95ab-1c0d-4fda-a111-11e16dff9ee2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c972ac6a-6853-4942-82e5-3650fc48791f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"074a0c9d-2541-426b-9501-6386614f5615","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-sub-8d991807","reconcileID":"e20c1eeb-9d0b-43ba-ac57-aaa10970a1d3","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-8d991807/e2e-shared-int-sub-8d991807","ai-tenant-e2e-mt-e9c8e886/e2e-shared-int-sub-8d991807","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T03:12:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2ab7651c-bf1d-47d5-babd-5267565a9035","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"01a05a36-7cfa-44cc-9a78-9bc6e3115d4c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-sub-8d991807","reconcileID":"e150a1b2-b5b8-4695-a91c-6b866b449118","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-sub-8d991807","reconcileID":"0794aae5-9033-44a9-8230-0dc61a0ec9ab","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-policy-8d991807","reconcileID":"8e630efa-f2da-4034-9bee-e0ee97847b72","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3c55c1d5-8c3a-4752-bbef-f73edfa21fdd","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"552ffbf0-3ccc-4f70-8af5-6e658e7ac606","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-sub-8d991807","reconcileID":"f89e51b8-119b-4add-abb1-f1a66ab54ef3","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-8d991807"}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"91961a66-8054-4f40-bb40-002839fbcca4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"namespace":"ai-tenant-e2e-mt-8d991807","name":"e2e-shared-int-sub-8d991807","reconcileID":"f89e51b8-119b-4add-abb1-f1a66ab54ef3","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-8d991807"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-e9c8e886/e2e-shared-int-sub-8d991807","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"79be8f98-e275-4e10-b81e-436a6a927ba2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-sub-8d991807","reconcileID":"62149822-cb49-4472-b6a0-e72b41274c5e","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3e1e22ba-9738-4779-98e8-642858142ced","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:39Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"588b948a-0d26-4ef0-93c0-618552ae3192","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-09T03:12:40Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:12:40Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d5c62374-115b-4b59-9481-e800b30f6422","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:12:40Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d5c62374-115b-4b59-9481-e800b30f6422","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-policy-8d991807","reconcileID":"ccc602da-a039-40dd-a9ff-fbf729029c44","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0a38cae7-5e7b-4b25-a4d8-094209668fd7","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0679f896-e724-4e20-bbf9-0b2401397c29","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-sub-8d991807","reconcileID":"121071d2-79cf-4bba-b92f-b6389b2f70ca","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-8d991807"}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a4aa43e5-d736-4b09-b0f1-2cf53e866452","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"namespace":"ai-tenant-e2e-mt-e9c8e886","name":"e2e-shared-int-sub-8d991807","reconcileID":"121071d2-79cf-4bba-b92f-b6389b2f70ca","MaaSSubscription":{"name":"e2e-shared-int-sub-8d991807","namespace":"ai-tenant-e2e-mt-e9c8e886"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f263af95-5d94-4c40-8b3f-2295adfcd3d2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:12:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"860d5709-0ee1-4b41-9d49-80d931de4e6e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:12:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"d15ccd3d-c94d-4a8f-b000-de9d186d23ea","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:12:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"f7e7fc51-78cf-408d-85f8-b6478a2b80ef","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:13:02Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-788622c4/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-09T03:13:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"917e6599-c6fb-46f7-9652-ec782fa60714","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:13:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"namespace":"ai-tenant-e2e-mt-788622c4","name":"e2e-policy-788622c4","reconcileID":"a00423d3-aefe-4b16-a0f8-68750b0ed10a","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:13:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"namespace":"ai-tenant-e2e-mt-788622c4","name":"e2e-policy-788622c4","reconcileID":"a00423d3-aefe-4b16-a0f8-68750b0ed10a"}
{"level":"info","ts":"2026-07-09T03:13:02Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"namespace":"ai-tenant-e2e-mt-788622c4","name":"e2e-policy-788622c4","reconcileID":"a00423d3-aefe-4b16-a0f8-68750b0ed10a","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:13:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1a28aaa5-14bf-45e6-8553-0ffadc0c9b11","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:13:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:13:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:13:23Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"namespace":"ai-tenant-e2e-mt-788622c4","name":"e2e-policy-788622c4","reconcileID":"3aab65f0-5f46-4edb-8094-0b5112b25f70","MaaSAuthPolicy":{"name":"e2e-policy-788622c4","namespace":"ai-tenant-e2e-mt-788622c4"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T03:13:25Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:13:25Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"4d88bc52-66a4-4786-8614-b17d7847a2f0","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:25Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"4d88bc52-66a4-4786-8614-b17d7847a2f0","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:33Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:13:33Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"a420b4e3-aa71-44d7-8c68-df5d1580dd50","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:33Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"a420b4e3-aa71-44d7-8c68-df5d1580dd50","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:33Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:13:33Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1dd5acf6-0e87-40ec-9ccb-7b154927e00f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:33Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1dd5acf6-0e87-40ec-9ccb-7b154927e00f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:33Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:13:33Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1dd5acf6-0e87-40ec-9ccb-7b154927e00f","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","tenantID":"e2e-inf-a-cf3835","subscriptionNamespace":"ai-tenant-e2e-inf-a-cf3835","gatewayName":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:13:33Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-cf3835","deploymentName":"maas-api-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:13:33Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-cf3835","deploymentName":"maas-api-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:13:33Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-cf3835","deploymentName":"maas-api-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:13:34Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:13:34Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-cf3835","deploymentName":"maas-api-e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:13:34Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1dd5acf6-0e87-40ec-9ccb-7b154927e00f","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:34Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:13:34Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"4b08bbce-bd6d-4ac6-bec8-37fb85296053","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:34Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"4b08bbce-bd6d-4ac6-bec8-37fb85296053","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:34Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:13:34Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1c2544c3-eec2-4714-a52e-c9704930b08f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:34Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1c2544c3-eec2-4714-a52e-c9704930b08f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:13:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:13:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"6cc572f3-560c-46c8-9e72-378d6844c5d1","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:13:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"7d5499cf-5c1f-49d1-b58e-0c33c8889c42","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:13:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:13:44Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"874fd208-ea00-44ad-ba9c-8bc64d5e43a4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:44Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"874fd208-ea00-44ad-ba9c-8bc64d5e43a4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:13:44Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"af56dcb4-5da8-4b3a-9418-f5cdd895c256","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:44Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"af56dcb4-5da8-4b3a-9418-f5cdd895c256","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:44Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:13:44Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"af56dcb4-5da8-4b3a-9418-f5cdd895c256","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","tenantID":"e2e-inf-b-defa37","subscriptionNamespace":"ai-tenant-e2e-inf-b-defa37","gatewayName":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:13:44Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-defa37","deploymentName":"maas-api-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:13:44Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-defa37","deploymentName":"maas-api-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:13:44Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-defa37","deploymentName":"maas-api-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:13:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:13:45Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-defa37","deploymentName":"maas-api-e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:13:45Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"af56dcb4-5da8-4b3a-9418-f5cdd895c256","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:13:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"4318a148-90a4-48fd-92a4-e85e17094a9c","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"4318a148-90a4-48fd-92a4-e85e17094a9c","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:13:45Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:13:45Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"650a1855-5676-406f-9dfc-71039e362fd4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:13:45Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"650a1855-5676-406f-9dfc-71039e362fd4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"13cfaa3e-5e57-4411-b770-f914f5e6db68","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"b1f894cf-1b56-41e0-9d01-162cf980c081","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"4f72043c-0aa1-4486-b0ed-14647ef7c731","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"error","ts":"2026-07-09T03:14:00Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"7313d5c1-5af7-40f9-9fe1-f370710cc0ab","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"test-model-cf3835-sub\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"87d44f0d-c0e0-45ea-84ab-c295178857ab","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"213c6219-e694-4d2b-8308-3468a82372be","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"test-model-cf3835","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-inf-a-cf3835/test-model-cf3835-sub"]}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"6cd61d48-b4af-4d7e-9d21-0984230b2023","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"195127f6-e91b-4ad5-abd7-35e023522b13","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"ac243001-4f35-4759-8a2e-96822ed9d773","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"08e6e805-353f-4ccb-a65b-9991087481fd","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"caadb903-0927-41ad-8539-683860641c0b","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-cf3835","tenantID":"e2e-inf-a-cf3835","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"ef1454ed-0aff-48af-99a2-45324a01204e","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"caadb903-0927-41ad-8539-683860641c0b"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"caadb903-0927-41ad-8539-683860641c0b","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"e2e-inf-a-cf3835-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"7c654a17-69e1-47cd-aa86-d7a4f1c953d0","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:00Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"3c1123ce-c996-45bf-a1ef-a55d4f3d2969","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-cf3835","tenantID":"e2e-inf-a-cf3835","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"3c1123ce-c996-45bf-a1ef-a55d4f3d2969"}
{"level":"info","ts":"2026-07-09T03:14:00Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"3c1123ce-c996-45bf-a1ef-a55d4f3d2969","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"e2e-inf-a-cf3835-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:04Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"a216dc8a-7d59-4147-b8f8-341adae411bf","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"136eeb2c-8508-4508-a262-ac40f0bbe164","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"a95b3491-0fd9-4320-8b12-0adb188f7805","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:05Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:14:05Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"e53f9177-6cb4-43cf-af81-17a06bed5980","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-cf3835","tenantID":"e2e-inf-a-cf3835","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:05Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"065bca27-ed53-48e7-ae85-5cf41847ebb2","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:05Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"e53f9177-6cb4-43cf-af81-17a06bed5980"}
{"level":"info","ts":"2026-07-09T03:14:05Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"e53f9177-6cb4-43cf-af81-17a06bed5980","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"e2e-inf-a-cf3835-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:14:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:14:10Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:14:10Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6a0d1565-db1d-4044-b511-70d45d50ec32","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:14:10Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6a0d1565-db1d-4044-b511-70d45d50ec32","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:14:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"04f3a526-57c6-4b8b-89dd-f010888798ab","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"0b182f2f-af59-4a6a-9a3c-0481e7fd2605","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c88913e4-5d61-4268-8e64-9581893fc076","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:19Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:14:19Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1537bfb9-8b59-4594-a4a8-00eef5b6ba28","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:14:19Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1537bfb9-8b59-4594-a4a8-00eef5b6ba28","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:14:19Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"4c3a794a-a242-464c-bc86-249fa5e43bb4","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:19Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:14:19Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"f626685b-fa06-41c9-88ec-77645dd5995b","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-a-cf3835","tenantID":"e2e-inf-a-cf3835","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"65cec2d0-acb3-4303-96ab-92b5bab3f066","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:19Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"f626685b-fa06-41c9-88ec-77645dd5995b"}
{"level":"info","ts":"2026-07-09T03:14:19Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"f626685b-fa06-41c9-88ec-77645dd5995b","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"e2e-inf-a-cf3835-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:14:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"50feb8a3-4f5a-4702-9396-8e5611c5d7f8","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:14:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"50feb8a3-4f5a-4702-9396-8e5611c5d7f8","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"4c3f72c7-c509-47ef-b108-475e58889918","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"d664afaf-34bf-4c4e-ab04-9b01650de4cc","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"12702712-fd73-4c19-b3bc-fd0eef959083","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"e5c28125-9055-4d1d-b127-55f90ccb4175","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"test-model-defa37","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-inf-b-defa37/test-model-defa37-sub"]}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"c95e86a9-1f5e-4699-984a-3d49923f532c","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"53a15df0-74be-425e-904a-1bde7909250d","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"3fb2935d-9294-464e-a68d-f0b9d914eac9","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"b25ad770-d939-4302-bc5f-5c1d19d77c24","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"1a683495-8c4b-4f7b-8a81-5c4f6523ab13","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-defa37","tenantID":"e2e-inf-b-defa37","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"0f579230-b5b6-4d7e-9354-f73b2eda26d2","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"1a683495-8c4b-4f7b-8a81-5c4f6523ab13"}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"1a683495-8c4b-4f7b-8a81-5c4f6523ab13","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"e2e-inf-b-defa37-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"dba2b61c-26a6-40e4-ade7-ea414a61fdba","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:36Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:14:37Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"0c8d603c-87fb-47d8-a0a1-163f5adaa379","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-defa37","tenantID":"e2e-inf-b-defa37","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:37Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"0c8d603c-87fb-47d8-a0a1-163f5adaa379"}
{"level":"info","ts":"2026-07-09T03:14:37Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"0c8d603c-87fb-47d8-a0a1-163f5adaa379","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"e2e-inf-b-defa37-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:37Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"8d578027-2e38-4361-8ad3-91c603ab2929","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:14:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:14:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"75718dfc-15e1-4c1c-92d1-513f48465c0e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"35906572-7165-47ad-b711-a871463b938a","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"d58a8182-5fa1-4a32-a280-7998ea89467f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"ebf3e2f2-cd05-4c9d-9e55-0315c290537c","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"5272e5c3-e8c3-44c8-a812-efc89687efb0","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"8fca276e-358b-41c6-9519-f87970c7f5ec","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:42Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"ee41378c-d864-48f3-915a-c110a715a03e","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-defa37","tenantID":"e2e-inf-b-defa37","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"8483be87-c94b-480f-9105-d718466709e5","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"ee41378c-d864-48f3-915a-c110a715a03e"}
{"level":"info","ts":"2026-07-09T03:14:42Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"ee41378c-d864-48f3-915a-c110a715a03e","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"e2e-inf-b-defa37-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"26bd63c5-fb7b-4f66-aad1-0431c122bf06","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"2cc41b23-ece2-4804-8765-d179c1fd5db7","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:14:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"14e0bfb6-42bc-48ee-aadb-ff0b8cf642af","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-inf-b-defa37","tenantID":"e2e-inf-b-defa37","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"381a6d08-93e8-4c11-ab65-1924485df14f","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"14e0bfb6-42bc-48ee-aadb-ff0b8cf642af"}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"14e0bfb6-42bc-48ee-aadb-ff0b8cf642af","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"e2e-inf-b-defa37-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"e1341406-123d-4640-a5de-6558119af8b3","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d56fcc19-30cb-482a-bff8-646b3f90fafe","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"92330ec5-1d53-41ef-89c7-95ca0e2e9f6d","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:14:55Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:14:55Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"8f52d981-0c8d-473f-8fc1-3a82abfc8d66","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:14:55Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"8f52d981-0c8d-473f-8fc1-3a82abfc8d66","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-auth","reconcileID":"54c859d6-6109-4a71-99a0-b0947e5e2374","MaaSAuthPolicy":{"name":"test-model-cf3835-auth","namespace":"ai-tenant-e2e-inf-a-cf3835"},"model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"0107a9d4-b963-422e-a104-3dcb70aa454a","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"9e319aa1-b3c0-4581-a7de-053e5f6004eb","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"8d016f58-c4da-4806-9ea8-ec4f052c7f73","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"629cfbce-3506-402c-a638-8073ff066726","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"55b72781-1aaf-4cfb-8542-1755eed9acb8","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835","subscription":"test-model-cf3835-sub"}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"74b269a5-e269-4557-a32b-83849a25d69a","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"55b72781-1aaf-4cfb-8542-1755eed9acb8","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835-sub","reconcileID":"55b72781-1aaf-4cfb-8542-1755eed9acb8","MaaSSubscription":{"name":"test-model-cf3835-sub","namespace":"ai-tenant-e2e-inf-a-cf3835"},"name":"maas-trlp-test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835","model":"ai-tenant-e2e-inf-a-cf3835/test-model-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"test-model-cf3835","reconcileID":"3dd51428-1425-4646-9c37-521f6a7c0d99","MaaSModelRef":{"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"},"routeName":"test-model-cf3835-kserve-route","namespace":"ai-tenant-e2e-inf-a-cf3835","llmisvcName":"test-model-cf3835","gateway":"openshift-ingress/e2e-inf-a-cf3835","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"747c7ebf-e115-438f-9ab7-5ef2d49889b8","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-auth","reconcileID":"0c78a82f-6185-4975-b063-f480951ad300","MaaSAuthPolicy":{"name":"test-model-defa37-auth","namespace":"ai-tenant-e2e-inf-b-defa37"},"model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37"}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"58d2dd5f-2289-4eb0-8f3e-845538e0cc55","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"c5ef7380-0fbc-4f7e-9688-fe4dd468886d","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"d3d56258-51b0-44dc-8647-2e7f47617c8b","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"76a73f60-9f4f-403c-8efb-6843aa3a6d88","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"14303928-aeed-44b8-9e08-6688eb9c9179","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37","subscription":"test-model-defa37-sub"}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"ae653af7-bab0-4de2-bf6f-f122db9157e2","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"14303928-aeed-44b8-9e08-6688eb9c9179","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37"}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37-sub","reconcileID":"14303928-aeed-44b8-9e08-6688eb9c9179","MaaSSubscription":{"name":"test-model-defa37-sub","namespace":"ai-tenant-e2e-inf-b-defa37"},"name":"maas-trlp-test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37","model":"ai-tenant-e2e-inf-b-defa37/test-model-defa37"}
{"level":"info","ts":"2026-07-09T03:15:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"test-model-defa37","reconcileID":"a886fa40-dfeb-4a76-8503-f2f979ef2dae","MaaSModelRef":{"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"},"routeName":"test-model-defa37-kserve-route","namespace":"ai-tenant-e2e-inf-b-defa37","llmisvcName":"test-model-defa37","gateway":"openshift-ingress/e2e-inf-b-defa37","hostnames":[]}
{"level":"info","ts":"2026-07-09T03:15:04Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"error","ts":"2026-07-09T03:15:04Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1998e8ba-c303-4a59-931c-c4dc55c072f9","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:04Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"1998e8ba-c303-4a59-931c-c4dc55c072f9","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:04Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","namespace":"ai-tenant-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","count":0}
{"level":"info","ts":"2026-07-09T03:15:04Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-cf3835/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-cf3835","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-cf3835","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-cf3835"},"tenantIdentifier":"e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","tenant":"e2e-inf-a-cf3835","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-09T03:15:04Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-cf3835","deploymentName":"maas-api-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","gvk":"Deployment","name":"maas-api-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","gvk":"Service","name":"maas-api-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:04Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cf3835"},"namespace":"ai-tenant-e2e-inf-a-cf3835","name":"default-tenant","reconcileID":"da79d740-5683-4bf6-a974-a03a7fe639df","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-a-cf3835"}
{"level":"info","ts":"2026-07-09T03:15:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:15:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:15:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"c2337884-014d-4d78-a258-c139eeb84f16","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:09Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"74e7a641-fdbe-4b06-943a-836d70cf07da","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:15Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:15:15Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"65e4d98b-c702-4228-b55f-a0c765d67dea","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:15Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"65e4d98b-c702-4228-b55f-a0c765d67dea","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c02ccf4f-1e39-4def-ba5a-1d96991b0085","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"9e8185ea-0f7e-49bd-9296-8c770c063fbe","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:15:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:15:40Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:15:40Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"e7505b71-99cc-4fdf-b5df-27beb2185c5e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:40Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"e7505b71-99cc-4fdf-b5df-27beb2185c5e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:43Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"error","ts":"2026-07-09T03:15:43Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"437c9870-eefb-4129-9183-31dd740d4a59","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:43Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"437c9870-eefb-4129-9183-31dd740d4a59","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:43Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","namespace":"ai-tenant-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","count":0}
{"level":"info","ts":"2026-07-09T03:15:43Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-b-defa37/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-b-defa37","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-b-defa37","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-b-defa37"},"tenantIdentifier":"e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","tenant":"e2e-inf-b-defa37","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-09T03:15:43Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-b-defa37","deploymentName":"maas-api-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","gvk":"Deployment","name":"maas-api-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","gvk":"Service","name":"maas-api-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:43Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-defa37"},"namespace":"ai-tenant-e2e-inf-b-defa37","name":"default-tenant","reconcileID":"ddbc08bc-9a14-48a0-bbb8-f0f5a9e29501","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-b-defa37"}
{"level":"info","ts":"2026-07-09T03:15:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"3caaed5f-b058-4cf4-86de-9846603331a2","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0cb0d2e1-2dfe-4e80-a2e9-4d20ee49566e","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:15:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"57dfdec3-c3f7-40ee-9ebd-16d13979ad67","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"57dfdec3-c3f7-40ee-9ebd-16d13979ad67","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:15:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"8d4e557e-4edf-432f-8413-bd4d52e0da66","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"8d4e557e-4edf-432f-8413-bd4d52e0da66","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:15:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"d4c17fbd-79f5-4a00-a264-9cbcd8807267","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:15:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"d4c17fbd-79f5-4a00-a264-9cbcd8807267","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:15:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"1fa7dd32-02a7-42bb-a535-26c8824732ad","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:15:54Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a3778000-4da2-4be0-b552-30d7b6ffaa4f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T03:16:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:16:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:16:25Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:16:25Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ced2d028-1c16-48ef-b632-355c658edd22","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:16:25Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ced2d028-1c16-48ef-b632-355c658edd22","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:16:38Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:16:38Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4e38cf3-87cc-4167-adb3-a6b8564fc8fa","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:16:38Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4e38cf3-87cc-4167-adb3-a6b8564fc8fa","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:16:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:16:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:17:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:17:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:17:10Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:17:10Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ad3c17fc-6247-4055-800e-95080d9e252e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:17:10Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ad3c17fc-6247-4055-800e-95080d9e252e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:17:23Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:17:23Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"3eab08df-762c-4642-b6e8-3739d2ef0119","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:17:23Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"3eab08df-762c-4642-b6e8-3739d2ef0119","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:17:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:17:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:17:55Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:17:55Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0a9c9794-2684-4c34-823c-573d3d4a06ab","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:17:55Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0a9c9794-2684-4c34-823c-573d3d4a06ab","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:18:08Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:18:08Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"364e4e3e-4744-4ef5-b159-d81a2704b575","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:18:08Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"364e4e3e-4744-4ef5-b159-d81a2704b575","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:18:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:18:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:18:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:18:38Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:18:40Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-09T03:18:40Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"85b83765-7e26-4b5f-a1da-733f1ceacef5","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:18:40Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"85b83765-7e26-4b5f-a1da-733f1ceacef5","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:18:53Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:18:53Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"0df69ff1-6659-4b7d-9b89-4d323c32cb3e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:18:53Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"0df69ff1-6659-4b7d-9b89-4d323c32cb3e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:19:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"error","ts":"2026-07-09T03:19:02Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"6ed83d23-d16c-4232-a99d-99177a336b8c","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T03:19:02Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"6ed83d23-d16c-4232-a99d-99177a336b8c","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T03:19:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","namespace":"ai-tenant-e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","count":0}
{"level":"info","ts":"2026-07-09T03:19:02Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-api-a-8d5c89/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-api-a-8d5c89","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-api-a-8d5c89","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-api-a-8d5c89"},"tenantIdentifier":"e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","tenant":"e2e-api-a-8d5c89","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-09T03:19:02Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-api-a-8d5c89","deploymentName":"maas-api-e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","gvk":"Deployment","name":"maas-api-e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","gvk":"Service","name":"maas-api-e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","gvk":"HTTPRoute","name":"maas-api-route-e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:02Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8d5c89"},"namespace":"ai-tenant-e2e-api-a-8d5c89","name":"default-tenant","reconcileID":"a4da07ea-8f7c-455c-a9be-52e7981ec0c5","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-api-a-8d5c89"}
{"level":"info","ts":"2026-07-09T03:19:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T03:19:08Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}