{"level":"info","ts":"2026-07-13T03:31:08Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-33013832"},"namespace":"ai-tenant-e2e-derive-33013832","name":"default-tenant","reconcileID":"4125d2d3-f697-4568-8d4f-9e743001e729","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-derive-33013832"}
{"level":"info","ts":"2026-07-13T03:31:12Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"9665a65c-4a84-40a6-b518-bfb388dcba60","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:31:12Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c0675bbc-a77f-4fad-a804-1514f83e1649","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c65642b8-0915-4ded-a70c-c49c59e1da28","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ab1dbd00-ecb2-423e-86fa-9cbdec088043","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:14Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-58412af8/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-policy-58412af8","reconcileID":"e2bfc6d9-c394-476d-b98c-126d0a817e4e","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-policy-58412af8","reconcileID":"e2bfc6d9-c394-476d-b98c-126d0a817e4e"}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-policy-58412af8","reconcileID":"e2bfc6d9-c394-476d-b98c-126d0a817e4e","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"cbe8ff5c-edb2-4c07-b6a9-53a13183473b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ada0adf5-3c94-4a62-b5ab-d38f934b49e8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T03:31:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-sub-58412af8","reconcileID":"0ad4184a-d886-4baf-a061-31c4ecac1ec0","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-sub-58412af8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ca9f73a0-7d3c-4c2f-a5e4-6976790668aa","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9142fa9c-6b2d-4214-868e-d678447514e3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-sub-58412af8","reconcileID":"9346e67b-13d8-4b3c-8889-95605e75903c","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-58412af8/e2e-sub-58412af8","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d5f7b5db-bd83-456f-acd4-03af60fd7cf0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"51a7e44a-6fd6-4d59-be53-667a4cd5b156","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-sub-58412af8","reconcileID":"0c229216-e4d5-4c28-b1e6-90a2710432b2","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-policy-58412af8","reconcileID":"c6cb3f56-9c94-4245-a1c8-9317b292f833","MaaSAuthPolicy":{"name":"e2e-policy-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e2df2b44-334d-454b-a788-0de27a3296cf","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"43fdf951-cc0d-4c52-b814-239330eb354f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-sub-58412af8","reconcileID":"2ce5cfa9-7bf9-4d15-8a39-64a86b3748fb","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-sub-58412af8"}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"45fce68f-4863-45d9-b9cf-814bc4231a85","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"namespace":"ai-tenant-e2e-mt-58412af8","name":"e2e-sub-58412af8","reconcileID":"2ce5cfa9-7bf9-4d15-8a39-64a86b3748fb","MaaSSubscription":{"name":"e2e-sub-58412af8","namespace":"ai-tenant-e2e-mt-58412af8"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9286305e-f0d9-4844-8154-a518ba9b663d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:15Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"44b1fb5d-0a7e-411a-9613-90128c11a122","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:31:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:31:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:31:19Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"1dec76d8-8c1c-4582-9954-c9ba411d119a","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:31:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2f2d3fd3-4460-499c-9343-54dad41a2eec","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"834e87d6-e5cc-4061-8d27-1c9346fb95b5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:22Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-c120cae6/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:31:22Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"namespace":"ai-tenant-e2e-mt-c120cae6","name":"e2e-policy-c120cae6","reconcileID":"8269d872-e3f0-4fd0-a82d-2edc120e7386","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:31:22Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"namespace":"ai-tenant-e2e-mt-c120cae6","name":"e2e-policy-c120cae6","reconcileID":"8269d872-e3f0-4fd0-a82d-2edc120e7386"}
{"level":"info","ts":"2026-07-13T03:31:22Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"namespace":"ai-tenant-e2e-mt-c120cae6","name":"e2e-policy-c120cae6","reconcileID":"8269d872-e3f0-4fd0-a82d-2edc120e7386","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:31:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"db2f2a1c-2c41-44e5-a266-4868ab1ca017","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:31:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:31:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"44a3e235-7ffd-4e08-8a6f-582178b7202f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:31:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"44a3e235-7ffd-4e08-8a6f-582178b7202f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:31:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:31:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:31:48Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"namespace":"ai-tenant-e2e-mt-c120cae6","name":"e2e-policy-c120cae6","reconcileID":"72559142-df8c-4831-b409-23fc42d5d524","MaaSAuthPolicy":{"name":"e2e-policy-c120cae6","namespace":"ai-tenant-e2e-mt-c120cae6"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:32:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:32:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:32:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:32:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bda818bb-5dc8-4c6d-b4aa-bb1a37a94cd3","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:32:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bda818bb-5dc8-4c6d-b4aa-bb1a37a94cd3","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9d0d09fb-89b2-4927-bd76-07192eb215a4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:26Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-5b92e04a/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"namespace":"ai-tenant-e2e-mt-5b92e04a","name":"e2e-policy-5b92e04a","reconcileID":"512aefdd-0e9a-4b36-b09a-a72c3f4cc66a","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"namespace":"ai-tenant-e2e-mt-5b92e04a","name":"e2e-policy-5b92e04a","reconcileID":"512aefdd-0e9a-4b36-b09a-a72c3f4cc66a"}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"namespace":"ai-tenant-e2e-mt-5b92e04a","name":"e2e-policy-5b92e04a","reconcileID":"512aefdd-0e9a-4b36-b09a-a72c3f4cc66a","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"474b650c-654a-442d-b182-06e7731fccd3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"namespace":"ai-tenant-e2e-mt-5b92e04a","name":"e2e-policy-5b92e04a","reconcileID":"20b52a2f-fbfa-4e8d-aafb-473056b7e386","MaaSAuthPolicy":{"name":"e2e-policy-5b92e04a","namespace":"ai-tenant-e2e-mt-5b92e04a"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:32:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"13bc4ba7-f2cc-4589-a368-27cf788835c6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c2a2fe48-d4ba-4422-9df3-cc319b49024e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"010c9569-c838-4ecf-b4b4-62be49528afe","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b5755bea-a90a-4682-bf0b-8e0d45ed4e06","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-376fe730/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-policy-376fe730","reconcileID":"c132057c-d0e5-44cf-94d6-658095c659a9","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-policy-376fe730","reconcileID":"c132057c-d0e5-44cf-94d6-658095c659a9"}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-policy-376fe730","reconcileID":"c132057c-d0e5-44cf-94d6-658095c659a9","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"32fa7dd6-36a5-4bc3-8d7a-b5d7c25d3f0e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d0a879e3-8acf-4c1e-9858-f84d46265b20","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"df3929f7-feb0-4c3c-8fe6-c4b1bb741b2c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T03:32:33Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-sub-376fe730","reconcileID":"5706dc87-b288-43af-aefa-bd07caf54885","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-sub-376fe730\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"18275771-6f40-485a-91e0-45b2b809e0e8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-sub-376fe730","reconcileID":"6b3c62e5-d397-4fc5-8657-c9dc5b3968c9","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-376fe730/e2e-shared-sub-376fe730","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c225f4da-f5cb-4b01-aec5-38f678bf902d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a460f0d6-352e-4cd5-9013-7f4b6af122b9","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:32:33Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-sub-376fe730","reconcileID":"92e3e8d9-22cc-4c52-9f5e-ad3e85d3bcd2","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5e47a1a8-fe68-4ef9-9eef-d436f3384a1b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:34Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-60a9a173/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e1a804c8-9d91-4a9d-ab02-d16d7851b394","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-policy-376fe730","reconcileID":"b7f78e59-42c0-488f-bc73-0dfaae90af71","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-policy-376fe730","reconcileID":"b7f78e59-42c0-488f-bc73-0dfaae90af71"}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-policy-376fe730","reconcileID":"b7f78e59-42c0-488f-bc73-0dfaae90af71","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2afca3a4-fa77-4be5-84db-25703d35c9a0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0e96f9d5-df36-42ef-8243-a77c11dafd72","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dd892539-86fb-4720-8c31-dc2188af0a95","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-sub-376fe730","reconcileID":"0129d1c3-667a-4860-9da7-b91e759a2057","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-376fe730/e2e-shared-sub-376fe730","ai-tenant-e2e-mt-60a9a173/e2e-shared-sub-376fe730","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"de7fb3b6-dc45-4336-bbcc-3b7fb43b60e8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d839882e-ae42-42dc-806c-e11ae2b34ce7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-sub-376fe730","reconcileID":"135786a3-6d67-4288-b35a-fa00de0f42ce","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T03:32:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-sub-376fe730","reconcileID":"b83a4ac4-e701-43c0-a570-2d8077c4ec3a","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-policy-376fe730","reconcileID":"b3c16c77-4292-44ae-a9ba-b75fdfcec29c","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7982acf0-75bd-4d00-a0ce-bc7b2fb9a47f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"62a988d5-baf8-469a-872f-8be249efbf3d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-sub-376fe730","reconcileID":"c549b501-4255-4aaf-85c8-2e4765f79c14","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-sub-376fe730"}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"12974e01-f076-41ac-b79a-dbfe683ab7fc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"namespace":"ai-tenant-e2e-mt-376fe730","name":"e2e-shared-sub-376fe730","reconcileID":"c549b501-4255-4aaf-85c8-2e4765f79c14","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-376fe730"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-60a9a173/e2e-shared-sub-376fe730","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"534870de-d7b1-4b15-8ad6-ec2cb1cc2b39","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c29cb19f-1495-43e8-8c2a-e0e8419c4e70","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f399ec4b-6d18-4d0a-9380-1fdb8cce10fb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-sub-376fe730","reconcileID":"95b60bca-2695-4c8c-acef-683d73f01d25","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:32:38Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"b40c5425-45a1-4af7-90ca-e84587e457d9","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:32:38Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"aef8208f-42db-437e-8cc2-861c72cacf8f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:32:38Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-sub-376fe730","reconcileID":"f65c98ec-acab-4172-82f7-39f757d3fad0","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-policy-376fe730","reconcileID":"abcb2986-f710-4c77-b751-94f775732f49","MaaSAuthPolicy":{"name":"e2e-shared-policy-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1b38f7ae-81a4-44c0-9bba-a5a071883c68","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d38b66ed-fc37-4e61-8c88-4a3f3a360c9a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-sub-376fe730","reconcileID":"c80c2b71-7302-4ad0-b66b-74e618a8524d","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-sub-376fe730"}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6c074603-228f-4f6e-9958-51ca7956d519","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"namespace":"ai-tenant-e2e-mt-60a9a173","name":"e2e-shared-sub-376fe730","reconcileID":"c80c2b71-7302-4ad0-b66b-74e618a8524d","MaaSSubscription":{"name":"e2e-shared-sub-376fe730","namespace":"ai-tenant-e2e-mt-60a9a173"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"765691a8-3964-434c-bf46-1744d4e933bb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:32:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"caaef58a-ec15-4542-8b06-9943e6e484e6","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:32:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a6064f5b-b518-4198-ad0b-dd5f4f304b6b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:32:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4e18246f-1aab-4150-beed-c36f0e3262d5","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:32:47Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"84141273-53d5-4f8d-9227-129efbda7ea7","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:32:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:32:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:32:51Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:32:51Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"1cd8b9e4-a986-4967-9fd3-f18e9b25a6a3","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:32:51Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"1cd8b9e4-a986-4967-9fd3-f18e9b25a6a3","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:32:51Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:32:51Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"70be62c1-5750-455e-af6d-1dad313a560f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:32:51Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"70be62c1-5750-455e-af6d-1dad313a560f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:32:52Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:32:52Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"ebf6ead5-0d51-43cf-9cae-1003bc598dae","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:32:52Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"ebf6ead5-0d51-43cf-9cae-1003bc598dae","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:32:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"8c586e80-2cc5-4dbd-a5b9-b1859441f3a3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:32:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"3a24faeb-7706-4ead-98e4-95389c24389b","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:33:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:33:05Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f580b10a-cca7-4e8a-a3b8-3922245bde46","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:33:05Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f580b10a-cca7-4e8a-a3b8-3922245bde46","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:33:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:33:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:33:36Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:33:36Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"e03453dc-4e57-414d-a4d5-72f351901c13","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:33:36Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"e03453dc-4e57-414d-a4d5-72f351901c13","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:33:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:33:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:33:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:33:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d0ec5a6a-fd69-4a71-add9-188055c605c4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:33:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d0ec5a6a-fd69-4a71-add9-188055c605c4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:34:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:34:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:34:21Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:34:21Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"45cb6e16-3597-4df1-837e-f8a2570e4ce1","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:34:21Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"45cb6e16-3597-4df1-837e-f8a2570e4ce1","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:34:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:34:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f1a55eee-7f14-4183-8c0f-574fac9e9186","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:34:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"f1a55eee-7f14-4183-8c0f-574fac9e9186","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:34:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:34:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:35:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:35:06Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"ba1d19f0-77a4-4e4e-a309-82aa51b431f9","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:35:06Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"ba1d19f0-77a4-4e4e-a309-82aa51b431f9","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:35:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:35:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:35:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:35:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"22de30fd-0ef2-4b95-8bdd-4b0daa02d2b4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:35:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"22de30fd-0ef2-4b95-8bdd-4b0daa02d2b4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:35:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:35:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:35:51Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:35:51Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"c12dbc0f-5373-4bcf-af9f-2c2e00c4b8fb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:35:51Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"c12dbc0f-5373-4bcf-af9f-2c2e00c4b8fb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"error","ts":"2026-07-13T03:36:01Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"12ddf95b-aab4-48ca-829c-d98cf15c3fdc","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:01Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"12ddf95b-aab4-48ca-829c-d98cf15c3fdc","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","namespace":"ai-tenant-e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","count":0}
{"level":"info","ts":"2026-07-13T03:36:01Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-a8f491a4/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-a8f491a4","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-a8f491a4","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-a8f491a4"},"tenantIdentifier":"e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","tenant":"e2e-mt-a8f491a4","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T03:36:01Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-a8f491a4","deploymentName":"maas-api-e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","gvk":"Deployment","name":"maas-api-e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","gvk":"Service","name":"maas-api-e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:01Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a8f491a4"},"namespace":"ai-tenant-e2e-mt-a8f491a4","name":"default-tenant","reconcileID":"bf6129e5-fb5e-4eda-8920-ec1acd5e84c0","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-a8f491a4"}
{"level":"info","ts":"2026-07-13T03:36:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:36:05Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6884ab4a-a7ed-40e2-b84b-71f9845aceb1","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:05Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"6884ab4a-a7ed-40e2-b84b-71f9845aceb1","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:36:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6af1f242-fa78-4df2-8878-92440c294fe2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4d0d3e85-7e5a-45f8-a004-f10e280a9b54","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-f22df0a4","reconcileID":"8dc0b47e-bdc1-4227-9bf2-57295ceba5fe","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-f22df0a4","reconcileID":"8dc0b47e-bdc1-4227-9bf2-57295ceba5fe"}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-f22df0a4","reconcileID":"8dc0b47e-bdc1-4227-9bf2-57295ceba5fe","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2a7dd50e-1b9d-47ed-a33a-73a664edbad4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-ns-f22df0a4","reconcileID":"0fee1804-d191-478c-943b-b4655b60e52d","MaaSAuthPolicy":{"name":"e2e-default-ns-f22df0a4","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d9f59014-e8f5-40ff-86fc-51686ddf0e3f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"826d8e08-8138-4e6b-af47-3ebd32738006","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f3976cb0-b9b7-41d3-b1d1-2f7d171f2eb3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:21Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"78d6baf5-0145-42bb-b71d-81ca49a5b0f6","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"2a2cc725-d999-4164-bba1-9a0d69d11fa2","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"2a2cc725-d999-4164-bba1-9a0d69d11fa2"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"2a2cc725-d999-4164-bba1-9a0d69d11fa2","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"25e9ef91-ccf7-43b2-9a54-1c0dcd0f5d64","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:21Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"3e55c40d-d4c5-477e-bfb5-ba7ff6503f55","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"3e55c40d-d4c5-477e-bfb5-ba7ff6503f55"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"3e55c40d-d4c5-477e-bfb5-ba7ff6503f55","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-gw-auth-c6a47af2","reconcileID":"2edadd49-82d7-48c8-bdc7-fb0aa261c296","MaaSAuthPolicy":{"name":"e2e-gw-auth-c6a47af2","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ad83acfd-4a18-42f8-8871-503dbb443ac0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f91d14d1-a90d-4d88-86bf-21b85513c0c2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"25af9cfd-7fa9-4872-8b0d-db265256142f","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"25af9cfd-7fa9-4872-8b0d-db265256142f","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"52bfad4b-c549-43dc-9606-e3a017f2d719","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"52bfad4b-c549-43dc-9606-e3a017f2d719","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:36:30Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"52bfad4b-c549-43dc-9606-e3a017f2d719","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","tenantID":"e2e-mt-766a483d","subscriptionNamespace":"ai-tenant-e2e-mt-766a483d","gatewayName":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-766a483d","deploymentName":"maas-api-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-766a483d","deploymentName":"maas-api-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-766a483d","deploymentName":"maas-api-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-766a483d","deploymentName":"maas-api-e2e-mt-766a483d"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"Reconciler error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"52bfad4b-c549-43dc-9606-e3a017f2d719","error":"Operation cannot be fulfilled on tenants.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"74915fe6-3991-41c1-b796-d58dbdcb95c1","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"74915fe6-3991-41c1-b796-d58dbdcb95c1","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:30Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"4833ff45-86d5-48d4-9ec7-bc98e00a1005","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:30Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"4833ff45-86d5-48d4-9ec7-bc98e00a1005","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0d7ec105-0b6d-4be4-a7ff-4a4c82bfe9cb","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:36:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"28bd9182-828a-4aae-83c8-e7672e4565f7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:36:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"d127e15f-5bfc-46f9-a346-714d5fde22e1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:36:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"19933175-d10b-41e1-a872-41e38e21c701","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:36:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:36:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:36:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:36:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0174f8e5-62c0-49f6-8811-993c85ad2970","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:36:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"0174f8e5-62c0-49f6-8811-993c85ad2970","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:36:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"6ed257e3-366c-47f5-8569-d38153efd624","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:36:56Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"cd44bb5c-dbcd-4450-9c41-40a2ba366c1f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:36:56Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"dbb3df17-404a-4556-a6fd-176dca040185","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:36:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"487ff4d6-4679-43df-bd72-20cc1acea5b4","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"fc44de4a-4294-4c57-bbc2-bf687573f97c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:01Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"71a4ee54-1a01-45b8-8ff4-2d61d3f23a0d","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"fdfda8c6-8aa3-4eeb-94de-548254becffb","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"468e9571-7281-46f4-b263-004b53e015d1","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"7e96f8cc-0408-46d2-9635-e6138ae36328","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"652e357a-011f-47b0-b2a9-a338f04173a8","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-766a483d","tenantID":"e2e-mt-766a483d","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"652e357a-011f-47b0-b2a9-a338f04173a8"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"gateway AuthPolicy created","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"652e357a-011f-47b0-b2a9-a338f04173a8","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"e2e-mt-766a483d-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"64361484-a702-4b14-b7f6-3985381be8ed","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"1777f584-3b67-4349-a0bc-ea0f66fec35f","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"4519ed84-5803-4489-acf1-3f3a14524f0b","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"error","ts":"2026-07-13T03:37:06Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"522167dc-ef0a-4574-bb38-b5e31a06ab0a","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-sub-766a483d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:06Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"12fafa7e-88a0-493e-b108-515f6fce353b","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-766a483d","tenantID":"e2e-mt-766a483d","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-13T03:37:06Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"4519ed84-5803-4489-acf1-3f3a14524f0b","name":"e2e-lifecycle-model-766a483d","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"e2e-lifecycle-model-766a483d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"103457e1-91b8-4923-a2f8-e04c8bb14cee","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"12fafa7e-88a0-493e-b108-515f6fce353b"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"12fafa7e-88a0-493e-b108-515f6fce353b","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"e2e-mt-766a483d-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"0a7fcd82-f4aa-469e-a5cd-5768d9ffc19e","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","model":"e2e-lifecycle-model-766a483d","subscriptionCount":1,"subscriptions":["ai-tenant-e2e-mt-766a483d/e2e-sub-766a483d"]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"82a7de74-a0cd-42ac-b12c-b116de532337","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:06Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"4c17e301-e4db-4110-bba8-751f036595d8","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:10Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"18747ef3-30e6-41a2-bd19-a703ddc8904e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"0e794935-999e-4209-be39-ded7c4d6902e","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"2e09fcb1-8d9d-4653-92e2-1febe56f08dc","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"d75648b7-7f4f-423e-9b8f-894753bbad70","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"f8b56c17-54e7-4544-86cb-f1c2a3987b6a","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"27c33d4a-8808-4396-851c-ada77e6a524a","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-766a483d","tenantID":"e2e-mt-766a483d","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"c39236c0-1e52-4d7a-8bc6-1af7138b23a2","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"27c33d4a-8808-4396-851c-ada77e6a524a"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"27c33d4a-8808-4396-851c-ada77e6a524a","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"e2e-mt-766a483d-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"b26fe25b-c3e3-4cb9-8be0-76c71ead9313","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"gatewayNamespace":"openshift-ingress","gatewayName":"e2e-mt-766a483d","tenantID":"e2e-mt-766a483d","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-13T03:37:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"6c0942f5-4f6c-4ebc-8b81-1744b44160bc","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"6c0942f5-4f6c-4ebc-8b81-1744b44160bc","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"b06a5930-a2ed-4f2b-aa70-9f53103a5553","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","namespace":"ai-tenant-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Deleting MaaSAuthPolicy","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"b26fe25b-c3e3-4cb9-8be0-76c71ead9313"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"b26fe25b-c3e3-4cb9-8be0-76c71ead9313","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"e2e-mt-766a483d-maas-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","count":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-mt-766a483d/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-mt-766a483d","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-mt-766a483d","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-mt-766a483d"},"tenantIdentifier":"e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","tenant":"e2e-mt-766a483d","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T03:37:11Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-mt-766a483d","deploymentName":"maas-api-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"4dfffef6-3740-480c-9c21-8bcbb11e1821","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","namespace":"ai-tenant-e2e-mt-766a483d","llmisvcName":"e2e-lifecycle-model-766a483d","gateway":"openshift-ingress/e2e-mt-766a483d","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","gvk":"Deployment","name":"maas-api-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","gvk":"Service","name":"maas-api-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"6e528e78-fabe-4827-a53e-afba6fd89dac","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","gvk":"HTTPRoute","name":"maas-api-route-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:11Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"default-tenant","reconcileID":"96af031b-592b-48de-9e81-c7a5c30f55c7","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-mt-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:12Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-policy-766a483d","reconcileID":"24842df4-1a34-4240-b583-01863f1f20a6","MaaSAuthPolicy":{"name":"e2e-policy-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:12Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"3bcdb63c-7652-4adf-99ba-e70f376dc651","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d","subscription":"e2e-sub-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:12Z","msg":"no active subscriptions for model, deleting TokenRateLimitPolicy","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"3bcdb63c-7652-4adf-99ba-e70f376dc651","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d"}
{"level":"info","ts":"2026-07-13T03:37:12Z","msg":"Deleting TokenRateLimitPolicy (no remaining parent subscriptions)","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-sub-766a483d","reconcileID":"3bcdb63c-7652-4adf-99ba-e70f376dc651","MaaSSubscription":{"name":"e2e-sub-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"name":"maas-trlp-e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d","model":"ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"3cbcf75b-e144-4b87-b709-cf1eda60e431","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"3cbcf75b-e144-4b87-b709-cf1eda60e431","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"3cbcf75b-e144-4b87-b709-cf1eda60e431","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"a21e4798-aafd-4755-843f-422da4f41b60","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"a21e4798-aafd-4755-843f-422da4f41b60","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"a21e4798-aafd-4755-843f-422da4f41b60","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"29a54bd7-727c-424e-800c-be2fe56a68e2","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"29a54bd7-727c-424e-800c-be2fe56a68e2","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"29a54bd7-727c-424e-800c-be2fe56a68e2","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"5923e61b-b99c-4d34-9ceb-e4e8c7cb44ad","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"5923e61b-b99c-4d34-9ceb-e4e8c7cb44ad","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"5923e61b-b99c-4d34-9ceb-e4e8c7cb44ad","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"3afe00fe-5d11-4e68-9c79-9ce621925819","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"3afe00fe-5d11-4e68-9c79-9ce621925819","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"3afe00fe-5d11-4e68-9c79-9ce621925819","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"35f666b0-46d7-4b57-96a8-3506b1c5f97c","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"35f666b0-46d7-4b57-96a8-3506b1c5f97c","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"35f666b0-46d7-4b57-96a8-3506b1c5f97c","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"bac6103e-1930-4d4d-a302-6210c2782c99","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"bac6103e-1930-4d4d-a302-6210c2782c99","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"bac6103e-1930-4d4d-a302-6210c2782c99","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:37:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"f575ae05-a886-4ca8-8aca-51db12222f01","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"f575ae05-a886-4ca8-8aca-51db12222f01","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:17Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"f575ae05-a886-4ca8-8aca-51db12222f01","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:18Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"841f2fe9-44e1-4758-9627-e2b9c8cd35d8","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:18Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"841f2fe9-44e1-4758-9627-e2b9c8cd35d8","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:18Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"841f2fe9-44e1-4758-9627-e2b9c8cd35d8","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:18Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"95b054f5-26d3-411b-9e14-baff42707e2d","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:18Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"95b054f5-26d3-411b-9e14-baff42707e2d","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:18Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"95b054f5-26d3-411b-9e14-baff42707e2d","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:21Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"6ef2a5e9-16b1-4832-9287-90dc92560ae7","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:21Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"6ef2a5e9-16b1-4832-9287-90dc92560ae7","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:21Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"6ef2a5e9-16b1-4832-9287-90dc92560ae7","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:26Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"695c0216-6d94-44ea-ac3f-3309413d0faa","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:26Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"695c0216-6d94-44ea-ac3f-3309413d0faa","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:26Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"695c0216-6d94-44ea-ac3f-3309413d0faa","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:26Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"803c7430-b43b-4dfb-ba61-e57f421e28b3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:27Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"1fa46fd7-8732-4a27-9164-658c66b65474","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:37:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d6788b6b-2cb4-495d-bf27-0e44ce4189e3","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"d6788b6b-2cb4-495d-bf27-0e44ce4189e3","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:36Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"237b11e9-e0f7-4293-a675-e04789a9bdab","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:36Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"237b11e9-e0f7-4293-a675-e04789a9bdab","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:36Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"237b11e9-e0f7-4293-a675-e04789a9bdab","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:37:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-13T03:37:54Z","msg":"HTTPRoute does not reference configured gateway","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"bafccc84-b144-4f8e-a912-a6c89738a571","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"routeName":"e2e-lifecycle-model-766a483d-kserve-route","routeNamespace":"ai-tenant-e2e-mt-766a483d","expectedGateway":"openshift-ingress/maas-default-gateway","foundGateway":"openshift-ingress/e2e-mt-766a483d","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).validateLLMISvcHTTPRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:113\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*llmisvcHandler).ReconcileRoute\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/providers_llmisvc.go:40\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:140\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:54Z","msg":"failed to reconcile HTTPRoute","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"bafccc84-b144-4f8e-a912-a6c89738a571","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:152\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:37:54Z","msg":"Reconciler error","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"},"namespace":"ai-tenant-e2e-mt-766a483d","name":"e2e-lifecycle-model-766a483d","reconcileID":"bafccc84-b144-4f8e-a912-a6c89738a571","error":"HTTPRoute ai-tenant-e2e-mt-766a483d/e2e-lifecycle-model-766a483d-kserve-route does not reference gateway (expected: openshift-ingress/maas-default-gateway, found: openshift-ingress/e2e-mt-766a483d). The LLMInferenceService must be configured to use openshift-ingress/maas-default-gateway","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:37:57Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"36caf8bb-e77a-4165-b718-eafd7a2dd51b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:37:58Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"19288ae6-62e5-4046-abbd-45e0d5a98d63","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"b3a06918-ae38-4b68-b992-6782dab77696","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:02Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"3646277a-0387-4199-9808-681530dfc17a","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7634c326-dc6a-4bca-b7b9-24894a3f4f80","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"942fc7bf-8d25-45a0-82ab-8d2c1c5d915c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:14Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-76ae5d","reconcileID":"387243be-7ab5-44e8-b960-e998880d959f","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-76ae5d","reconcileID":"387243be-7ab5-44e8-b960-e998880d959f"}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-76ae5d","reconcileID":"387243be-7ab5-44e8-b960-e998880d959f","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"91391b91-780c-43c8-809c-fdb2f96d487e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ee2cca1b-5453-4ffd-9ed0-76fa1ab13b46","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4613b377-22d6-4471-874f-48c685f7e1f1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T03:38:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-76ae5d","reconcileID":"2b895e75-5911-49a8-a271-ae288f4d4fb6","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-default-int-sub-76ae5d\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5a66d4e8-4994-40d8-b840-2e7efef755d1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-76ae5d","reconcileID":"941584c5-0657-4c34-bf5d-d2b78c9d2e3d","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-default-int-sub-76ae5d","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ac0ac8ee-102f-4be8-83fc-0a28bc67ff19","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"81c07f3c-0202-4a5d-bea4-e3acf8461b9e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:38:14Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-76ae5d","reconcileID":"a6f69419-aa12-40db-8d11-bb53c2ecdcce","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-auth-76ae5d","reconcileID":"ff449b24-daae-4ccd-846e-4218164db334","MaaSAuthPolicy":{"name":"e2e-default-int-auth-76ae5d","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"549a3752-4a26-49ed-ba4e-d13627d789b5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dd0d4da8-5f4b-4c76-8035-77c88bb5beab","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-76ae5d","reconcileID":"5cfb6700-2b87-4d16-9991-af7fd16bbc3c","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-default-int-sub-76ae5d"}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b07eb462-16f8-44ab-afbe-ecd352304be9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-default-int-sub-76ae5d","reconcileID":"5cfb6700-2b87-4d16-9991-af7fd16bbc3c","MaaSSubscription":{"name":"e2e-default-int-sub-76ae5d","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"73d5c459-8d22-4ba2-8f56-bcd3762a4e49","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"4a0a33be-5bed-4c44-b051-72ffce80bae0","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b421b385-37b6-4770-972a-fa1a865c3704","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e44cf715-91e4-4dfa-91a9-55cacdf44813","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-1b2c7e07/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"84a4d141-dcbb-4285-9ebb-22a0cf573233","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"84a4d141-dcbb-4285-9ebb-22a0cf573233"}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"84a4d141-dcbb-4285-9ebb-22a0cf573233","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"675ae565-83d7-4dcc-aaf7-54b193a02bf5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"55ab1cb5-8db8-441f-bc20-4902d55b829a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dae9429c-6250-471c-9a4e-9399bec289d3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T03:38:15Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"e098ce96-39ff-43aa-967c-8e39418e85f7","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-1b2c7e07\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1a4f81e8-67bb-49c3-a8b8-dc8e86a5506c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"5320eff9-bb90-4037-8f2f-eb8460f837f7","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-1b2c7e07/e2e-shared-int-sub-1b2c7e07","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"2d9b504a-14e7-4a1d-a169-c08bf7b6e314","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:15Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"03570bd5-a92d-4cd3-a45a-744001f74e39","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"e4a0c9c1-3832-4bbe-81f6-a2b3689159f4","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"daf04d1b-c191-4e52-98bc-7fe985df0276","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"840e798d-1df1-489f-a864-fe33c81f188b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:16Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-651f3984/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"a22b9665-a082-4393-8fac-a6836d83ede3","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"a22b9665-a082-4393-8fac-a6836d83ede3"}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"a22b9665-a082-4393-8fac-a6836d83ede3","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f5c4b8ce-fc7c-4a8f-8ee0-a31f99921604","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5d68b98b-aa3a-4a3f-8c38-b5271f402612","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3c3e95e0-44d9-45f2-928b-69cd189eb82c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-13T03:38:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"53164df6-f155-422e-bd3d-f229d6ed1bbf","error":"Operation cannot be fulfilled on maassubscriptions.maas.opendatahub.io \"e2e-shared-int-sub-1b2c7e07\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"689d47ef-4c47-489e-a8bf-90e941f24c65","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"8b048f31-983c-4f8d-87df-63fb06b0eda2","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3,"subscriptions":["ai-tenant-e2e-mt-1b2c7e07/e2e-shared-int-sub-1b2c7e07","ai-tenant-e2e-mt-651f3984/e2e-shared-int-sub-1b2c7e07","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1a7edc60-e03b-4f45-9210-14dfba2e0821","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"05c8d5b9-b462-4217-99a5-abbb710d567f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"cc5cc35d-7eca-4a85-9a98-986418b5563e","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T03:38:16Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"3bff8054-8a5f-4c93-90d1-8a7929e57322","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":3}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"154734cc-857a-4c1b-a021-b83cc0e1fa84","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"674238b6-9ebd-461c-82a5-5bbc6c42d668","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4f8cdecd-9b3a-4e2e-a5b0-7f17a626f948","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"ad85f27a-47c9-465d-92bd-2828da5d0906","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-1b2c7e07"}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"420e34dc-980b-461a-9427-f4c09408b892","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"namespace":"ai-tenant-e2e-mt-1b2c7e07","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"ad85f27a-47c9-465d-92bd-2828da5d0906","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-1b2c7e07"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["ai-tenant-e2e-mt-651f3984/e2e-shared-int-sub-1b2c7e07","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0452ea96-f753-433c-8b24-a28c67478dd3","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"89d84fbc-718c-4676-b627-1da863687042","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:38:17Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"a5652433-dba4-4afe-844a-060cd069d4b2","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2}
{"level":"info","ts":"2026-07-13T03:38:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:38:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:38:20Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"9ab9e61c-b0f8-4cca-95b8-bc4464b6278c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:38:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"1822c0b7-ac45-4d37-94bb-fa8ac4af48b6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:38:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"1822c0b7-ac45-4d37-94bb-fa8ac4af48b6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:38:23Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-policy-1b2c7e07","reconcileID":"9e76007b-0f65-42f3-b2d3-f6cc0f568cf2","MaaSAuthPolicy":{"name":"e2e-shared-int-policy-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:38:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"4b01fa34-9e54-4cc0-bb28-c39e38697b60","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ddcee6dc-1615-4ca7-ae33-c5c973bdedeb","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:24Z","msg":"Rebuilding TokenRateLimitPolicy without deleted subscription","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"dab4bee1-3dea-40ad-b439-42421af2b405","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"model":"llm/facebook-opt-125m-simulated","subscription":"e2e-shared-int-sub-1b2c7e07"}
{"level":"info","ts":"2026-07-13T03:38:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fb154eea-5083-4e9a-80da-d0cd7e237752","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:24Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"namespace":"ai-tenant-e2e-mt-651f3984","name":"e2e-shared-int-sub-1b2c7e07","reconcileID":"dab4bee1-3dea-40ad-b439-42421af2b405","MaaSSubscription":{"name":"e2e-shared-int-sub-1b2c7e07","namespace":"ai-tenant-e2e-mt-651f3984"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-13T03:38:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b8d87e70-252f-4556-affd-783f764667d0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:24Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"85f6ecd1-5dc1-4bd4-b450-ec8d3ee5b23c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:27Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"52b3c997-978f-40e3-a948-6ce5a9482ced","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"c32d1b20-3874-4e88-918c-e52bee4f2fbe","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:35Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"00f417bc-86bc-4206-826e-0c8ac62aa16f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-13T03:38:40Z","logger":"TenantIdentifierFor","msg":"Using legacy/default tenant identifier (empty string)","tenant":"ai-tenant-e2e-mt-8ff9d671/default-tenant","labels":null,"reason":"no AITenant label"}
{"level":"info","ts":"2026-07-13T03:38:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"72a73aa0-f127-4a2b-80c1-c6bd8e0ff0e8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:40Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"namespace":"ai-tenant-e2e-mt-8ff9d671","name":"e2e-policy-8ff9d671","reconcileID":"7cfe069c-c47a-462d-b4f9-1aa57fc9802e","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-13T03:38:40Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"namespace":"ai-tenant-e2e-mt-8ff9d671","name":"e2e-policy-8ff9d671","reconcileID":"7cfe069c-c47a-462d-b4f9-1aa57fc9802e"}
{"level":"info","ts":"2026-07-13T03:38:40Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"namespace":"ai-tenant-e2e-mt-8ff9d671","name":"e2e-policy-8ff9d671","reconcileID":"7cfe069c-c47a-462d-b4f9-1aa57fc9802e","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-13T03:38:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"603820d9-2ac3-4cf5-95a5-7cf1b78d09f2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-13T03:38:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:38:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:39:01Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"namespace":"ai-tenant-e2e-mt-8ff9d671","name":"e2e-policy-8ff9d671","reconcileID":"e221d41d-858f-4854-aa47-f856d4774f85","MaaSAuthPolicy":{"name":"e2e-policy-8ff9d671","namespace":"ai-tenant-e2e-mt-8ff9d671"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-13T03:39:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:39:05Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7d6d9b74-8b02-4e8c-b48e-b1906ce8a9bb","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:39:05Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"7d6d9b74-8b02-4e8c-b48e-b1906ce8a9bb","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:39:10Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:39:10Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"8eba8725-da47-40c9-8ae4-ca10e474ccd4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:39:10Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"8eba8725-da47-40c9-8ae4-ca10e474ccd4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:39:10Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:39:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"8cd8e41b-7ac8-4bb9-96d4-9cd4ed34c562","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:39:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"8cd8e41b-7ac8-4bb9-96d4-9cd4ed34c562","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:39:11Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:39:11Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"4e3873c1-479c-45f3-a4de-2617610e9826","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:39:11Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"4e3873c1-479c-45f3-a4de-2617610e9826","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:39:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:39:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:39:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:39:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:39:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:39:50Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ee6f8eba-1fdc-4e6f-8dbe-920765b3a5e7","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:39:50Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"ee6f8eba-1fdc-4e6f-8dbe-920765b3a5e7","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:39:55Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:39:55Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"3eb75275-38e0-4387-8188-f523b8dd86cf","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:39:55Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"3eb75275-38e0-4387-8188-f523b8dd86cf","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:40:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:40:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:40:35Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:40:35Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bb429eeb-56d7-4729-a1d5-b8f8f89573a6","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:40:35Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"bb429eeb-56d7-4729-a1d5-b8f8f89573a6","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:40:40Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:40:40Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"d22f68b1-3ce0-40d9-8e18-995fff485e08","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:40:40Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"d22f68b1-3ce0-40d9-8e18-995fff485e08","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:40:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:40:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:41:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:41:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:41:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:41:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"9bca954c-bf5f-430a-9683-6f54809d40e4","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:41:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"9bca954c-bf5f-430a-9683-6f54809d40e4","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:41:25Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:41:25Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"57ee4813-0c2f-467a-b06f-debebf97f7a7","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:41:25Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"57ee4813-0c2f-467a-b06f-debebf97f7a7","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:41:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:41:47Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:42:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"error","ts":"2026-07-13T03:42:05Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"b55997c0-0da7-465d-b5a8-75436649539e","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:42:05Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"b55997c0-0da7-465d-b5a8-75436649539e","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:42:10Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:42:10Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"52583e86-3ff7-420e-98e5-3f70374fdd0a","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:42:10Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"52583e86-3ff7-420e-98e5-3f70374fdd0a","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:42:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:42:17Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-13T03:42:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"error","ts":"2026-07-13T03:42:20Z","msg":"failed to check database Secret","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"ffe47d83-f65a-475a-ab34-9380c9424011","name":"maas-db-config","namespace":"odh-ai-gateway-infra","error":"unable to get: odh-ai-gateway-infra/maas-db-config because of unknown namespace for the cache","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.checkDatabaseSecret\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:132\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:51\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-13T03:42:20Z","msg":"MaaS prerequisite error","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"ffe47d83-f65a-475a-ab34-9380c9424011","check":"database-secret","message":"failed to check database Secret 'maas-db-config' in namespace 'odh-ai-gateway-infra' due to a cluster API error","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile.CollectPrerequisiteReport\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/platform/tenantreconcile/prerequisites.go:53\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).checkDependenciesAndPrerequisites\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:297\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_reconcile.go:155\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*TenantReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/tenant_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-13T03:42:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Cleaning up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","namespace":"ai-tenant-e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Cleaned up MaaSAuthPolicy CRs","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","count":0}
{"level":"info","ts":"2026-07-13T03:42:20Z","logger":"TenantIdentifierFor","msg":"Resolved tenant identifier from AITenant label","tenant":"ai-tenant-e2e-inf-a-3eb586/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"e2e-inf-a-3eb586","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"e2e-inf-a-3eb586","maas.opendatahub.io/tenant-namespace":"ai-tenant-e2e-inf-a-3eb586"},"tenantIdentifier":"e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Cleaning up per-tenant maas-api resources","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","tenant":"e2e-inf-a-3eb586","namespace":"odh-ai-gateway-infra"}
{"level":"info","ts":"2026-07-13T03:42:20Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"e2e-inf-a-3eb586","deploymentName":"maas-api-e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","gvk":"Deployment","name":"maas-api-e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","gvk":"Service","name":"maas-api-e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","gvk":"HTTPRoute","name":"maas-api-route-e2e-inf-a-3eb586"}
{"level":"info","ts":"2026-07-13T03:42:20Z","msg":"Deleted tenant resource","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-3eb586"},"namespace":"ai-tenant-e2e-inf-a-3eb586","name":"default-tenant","reconcileID":"836fe623-ef7b-40c2-aab2-9bd4f94200e8","gvk":"CronJob","name":"maas-api-key-cleanup-e2e-inf-a-3eb586"}