{"level":"info","ts":"2026-07-09T22:45:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"64ebaa23-de02-4042-8192-b2333a23f53e","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"3f18010a-df3a-4e36-b72a-721a7bcb8a01","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"da6ad1a6-77ca-40d5-bb41-15749ee1c2ad","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"40369385-0120-405e-a281-1b2021720701","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:53Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"f228473d-4f3d-46d0-ac7c-923d3a0e853c","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"f228473d-4f3d-46d0-ac7c-923d3a0e853c"}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"f228473d-4f3d-46d0-ac7c-923d3a0e853c","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9c3041a9-9707-4d35-8e36-58e9b3984c71","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:53Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"a8fcb877-2548-4b21-b808-0a9205022162","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fa1c4563-e873-44e8-8b6c-13c931f31a3e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"29f77e21-25d6-4107-b5b9-e999e6af6965","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"a8fcb877-2548-4b21-b808-0a9205022162"}
{"level":"info","ts":"2026-07-09T22:45:53Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"a8fcb877-2548-4b21-b808-0a9205022162","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:45:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"4bdc264a-0178-434e-8625-810426ef35ae","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:45:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"4bdc264a-0178-434e-8625-810426ef35ae","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:45:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"f65b14f2-d048-4c26-9fb5-0c5d00997293","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:45:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"f65b14f2-d048-4c26-9fb5-0c5d00997293","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"de606e75-5f21-41c0-b621-83881acb8f48","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"de606e75-5f21-41c0-b621-83881acb8f48","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"1e1ae1fc-9c41-4423-b9c5-266d2817437c","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"1e1ae1fc-9c41-4423-b9c5-266d2817437c","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e906da2e-b208-4831-aa37-bdefcb305dc7","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e906da2e-b208-4831-aa37-bdefcb305dc7","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"821e7c84-5bc3-4df0-8e2f-a1b9e9dea387","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"821e7c84-5bc3-4df0-8e2f-a1b9e9dea387","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d7dc8dce-9658-4ce4-bdd7-5e651dd222f2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:10Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"9a56f129-3323-4c9c-9ffe-8b2f640ef2d1","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-neg-ghost-sub-f49cfd0a","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T22:46:10Z","msg":"model/route not found during cleanup, deleting TokenRateLimitPolicy via labels","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"9a56f129-3323-4c9c-9ffe-8b2f640ef2d1","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-f49cfd0a","error":"MaaSModelRef not found: llm/nonexistent-model-f49cfd0a"}
{"level":"info","ts":"2026-07-09T22:46:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b1245287-1dc8-42f0-884d-6aad3226f020","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ec897cdc-455b-4172-9181-9bc7cb5f5bea","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:46:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"90fbf9f0-428f-435c-8ee1-8673bfc6d9c7","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"90fbf9f0-428f-435c-8ee1-8673bfc6d9c7","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:46:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:46:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"a18569b7-24cb-4232-ab7a-b5bf3cbe569c","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"a18569b7-24cb-4232-ab7a-b5bf3cbe569c","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"9c79fb51-a767-4fc9-b6ee-f517713fa519","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"9c79fb51-a767-4fc9-b6ee-f517713fa519","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"a53469e1-7f30-4445-86db-89366c8027d6","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"a53469e1-7f30-4445-86db-89366c8027d6","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"99763807-37da-4fa5-b379-cd96a4dbb6dc","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"99763807-37da-4fa5-b379-cd96a4dbb6dc","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:25Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"014d4a12-9172-478f-905c-34eac06464b9","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"error","ts":"2026-07-09T22:46:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"9e132696-2c2d-44ee-ab15-5d11eade0ca8","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"9e132696-2c2d-44ee-ab15-5d11eade0ca8","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"44f5c004-7972-4aaf-83ce-b86b4a1b082f","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"44f5c004-7972-4aaf-83ce-b86b4a1b082f","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"f79c7289-6f42-4761-8416-7bcbafec94d9","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"f79c7289-6f42-4761-8416-7bcbafec94d9","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"13b5173b-0cae-4b40-8b0e-a6bd903ff4a6","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T22:46:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"7cfe2df2-fb50-4ccb-86eb-2860fef8ad45","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"7cfe2df2-fb50-4ccb-86eb-2860fef8ad45","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"c41a60fb-9de3-45a8-8bb4-5e6ee12bf215","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"c41a60fb-9de3-45a8-8bb4-5e6ee12bf215","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f49cfd0a","reconcileID":"138c15bc-cda5-4f05-b2a0-248d98d3d6c3","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f49cfd0a","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9b6669ba-e36a-4345-8eb3-ae3321341271","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b9d57e60-800e-4105-9911-63e13f466502","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"12e0b411-0caa-419e-aa35-aedbdffb02e0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9104d952-93a4-4b69-9903-0da10727962e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:40Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"2f8d4bd1-cc33-4b26-a9bc-69e381f32418","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"2f8d4bd1-cc33-4b26-a9bc-69e381f32418"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"2f8d4bd1-cc33-4b26-a9bc-69e381f32418","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"model not found, cleaning up generated AuthPolicy","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"2f8d4bd1-cc33-4b26-a9bc-69e381f32418","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-147cdb13"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1185454f-1483-4706-99b7-569fa3d5ec99","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:40Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"10be804f-a78a-4b35-9f04-4bf14c665128","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"10be804f-a78a-4b35-9f04-4bf14c665128"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"10be804f-a78a-4b35-9f04-4bf14c665128","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:46:40Z","msg":"model not found, cleaning up generated AuthPolicy","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"10be804f-a78a-4b35-9f04-4bf14c665128","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-147cdb13"}
{"level":"info","ts":"2026-07-09T22:46:41Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"f668c235-ef80-47ec-ae0a-c1f828b65aa4","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T22:46:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8b3a616c-eabd-4978-abe9-2ca7198a4e80","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:41Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-147cdb13","reconcileID":"f668c235-ef80-47ec-ae0a-c1f828b65aa4","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-147cdb13","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-147cdb13"}
{"level":"info","ts":"2026-07-09T22:46:41Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"75a809bc-8820-41d0-abe8-53c04b895c17","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:46:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"f6051885-d90d-4ecb-861b-fee5549a2a24","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"f6051885-d90d-4ecb-861b-fee5549a2a24","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:43Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"710f7933-c027-4ac1-a23d-a1dbfccc4fa5","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T22:46:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:46:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:46:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e1472d93-625a-4dfe-aed2-2224d09e269f","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e1472d93-625a-4dfe-aed2-2224d09e269f","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"ae1e3e8b-b186-4717-98e4-8e15ac3091ad","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"ae1e3e8b-b186-4717-98e4-8e15ac3091ad","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"f2b02511-0ff3-4f85-946c-7fc5c08992e3","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"f2b02511-0ff3-4f85-946c-7fc5c08992e3","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:46:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f0bfea60-4a84-4397-90de-c44de33c1c70","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:46:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"36b5416a-c618-4e07-869f-a47f6a814970","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:46:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"6fe2a0ac-705f-4ca4-828b-b0b03b9f9484","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"6fe2a0ac-705f-4ca4-828b-b0b03b9f9484","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"1fe2db85-bdf0-4aad-bdd3-266ac8f77996","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:46:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"1fe2db85-bdf0-4aad-bdd3-266ac8f77996","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"b60c9b2e-294e-42c6-bd58-c7ec91ee35ac","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"b60c9b2e-294e-42c6-bd58-c7ec91ee35ac","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"3c1a3507-ca01-459b-9abd-15278747d533","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"3c1a3507-ca01-459b-9abd-15278747d533","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"79d757f5-9add-47c6-8d10-0929e1ee765f","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"79d757f5-9add-47c6-8d10-0929e1ee765f","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"cf5273fa-3c8c-4db8-89fe-3de176ce963b","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"cf5273fa-3c8c-4db8-89fe-3de176ce963b","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3548f926-0f9b-44ff-b76f-1f8bcb80844a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:10Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"8911d5d9-39e8-459e-84fb-3ac637d2d824","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-apikey-sub-binding-9f62be12","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-09T22:47:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"128826ec-68f0-40d0-a5a3-04c445d36569","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fa54975d-0192-4f54-9b22-a46001de1cbe","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:47:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"fdf856b8-f48d-4eba-afbb-9e455ee54d8b","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"fdf856b8-f48d-4eba-afbb-9e455ee54d8b","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:47:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:47:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"bb611774-a702-4843-a941-bf079c1cb637","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"bb611774-a702-4843-a941-bf079c1cb637","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"21d391bc-c47a-4e86-9b45-a28400f0c717","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"21d391bc-c47a-4e86-9b45-a28400f0c717","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"93287daa-230c-41cb-9593-ae768c866922","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"93287daa-230c-41cb-9593-ae768c866922","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"4733f4ed-c93d-405d-9aac-59ad53e8f981","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"4733f4ed-c93d-405d-9aac-59ad53e8f981","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"acb48110-5d79-4794-ae63-55efd9e33c28","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"acb48110-5d79-4794-ae63-55efd9e33c28","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:28Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"f44c8477-f574-447e-987f-70dc9dd6da31","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"error","ts":"2026-07-09T22:47:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"c576e1d7-8fb1-4174-945e-a48fd972aafd","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"c576e1d7-8fb1-4174-945e-a48fd972aafd","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"94965da5-604b-4470-87f7-860176668c33","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"94965da5-604b-4470-87f7-860176668c33","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a99a12b1-9855-4f0a-9f1a-0e7b983a8afd","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T22:47:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"e4b5c4bc-5864-4131-af74-45ce104d59d9","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"e4b5c4bc-5864-4131-af74-45ce104d59d9","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"640d8d8f-9078-4566-aacd-947d1f828e4d","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"640d8d8f-9078-4566-aacd-947d1f828e4d","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"dc3750d6-083b-4f7e-8479-4a219d8f9dcb","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"823e6c43-5a83-4f49-906a-4a9926a8a487","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:42Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:47:42Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"61503029-ed65-4cb9-9426-515e5f570bd9","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:47:42Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"61503029-ed65-4cb9-9426-515e5f570bd9"}
{"level":"info","ts":"2026-07-09T22:47:42Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"61503029-ed65-4cb9-9426-515e5f570bd9","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:47:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"b87aac75-3224-41a2-ac8e-6f39ee07fd3c","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:47:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:47:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"443b06b1-b5ac-4cfc-b32c-a920e4cfe4b5","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"443b06b1-b5ac-4cfc-b32c-a920e4cfe4b5","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:46Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"b5c6573c-9513-448f-b270-5332ffb2a274","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-09T22:47:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"ba9ca71b-5dfb-4a78-a0fd-2b1e36e19245","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"2f25192b-0987-4018-8100-230a32d9b79d","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:47:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"cec38a56-86ba-47ea-8a9b-e26fb5cb365a","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"cec38a56-86ba-47ea-8a9b-e26fb5cb365a","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"f7344c9c-0465-4afa-bbbf-be5783c7814f","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"f7344c9c-0465-4afa-bbbf-be5783c7814f","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:49Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"e24356c9-02b5-4bf9-bdeb-934703882ed4","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"4e996153-d794-4fb7-8947-ff736d00d871","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"d0416f78-752d-45b3-ac3c-f5bf56cd4481","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"591a362f-d19f-48a1-88c3-9ed551519526","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"591a362f-d19f-48a1-88c3-9ed551519526"}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"591a362f-d19f-48a1-88c3-9ed551519526","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"086a55a1-1c91-4973-8223-caf8d315f940","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"dfae8743-1bf7-4564-b55f-6a9973c4bafc","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"dfae8743-1bf7-4564-b55f-6a9973c4bafc"}
{"level":"info","ts":"2026-07-09T22:47:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"dfae8743-1bf7-4564-b55f-6a9973c4bafc","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:47:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"deaf11bd-5a21-4e26-b40a-dda1cf8a25ee","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"deaf11bd-5a21-4e26-b40a-dda1cf8a25ee","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"3811fd4b-5d07-4883-92d5-d179c7ef8975","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"235a8548-86c7-4e19-a6a9-d1f03621bc89","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:47:54Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:47:54Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"a60b1701-0c16-4674-a4c0-60a213f06c45","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:47:54Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"a60b1701-0c16-4674-a4c0-60a213f06c45"}
{"level":"info","ts":"2026-07-09T22:47:54Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"a60b1701-0c16-4674-a4c0-60a213f06c45","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:47:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"8b504d0d-d999-4d49-9105-086c6dce409a","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"8b504d0d-d999-4d49-9105-086c6dce409a","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:47:55Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"afd72fc7-c799-4317-8e3a-8f0e4f4367cb","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T22:47:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"df0ec7a8-9cce-4786-9140-31387f0b4ca3","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:47:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"df0ec7a8-9cce-4786-9140-31387f0b4ca3","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"c2d8b240-4652-499c-96fb-a2541fc22cfe","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"c2d8b240-4652-499c-96fb-a2541fc22cfe","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"8730bd47-01d0-4af5-8ee9-7e352fa0a15e","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"8730bd47-01d0-4af5-8ee9-7e352fa0a15e","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"2218ff1a-89ad-4b32-a7cc-985f0c8d6961","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"2218ff1a-89ad-4b32-a7cc-985f0c8d6961","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"d6609961-2c37-4bec-9d19-c9717cbd5244","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"d6609961-2c37-4bec-9d19-c9717cbd5244","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:48:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:48:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"3d1e6bae-55fe-4674-88bc-9cd171d063f9","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"3d1e6bae-55fe-4674-88bc-9cd171d063f9","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"af8c7d5a-442c-4c5e-b143-10faa64ee0cc","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:13Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"491a6544-4c98-4e69-acc3-a08947483075","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated","model":"e2e-unconfigured-facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/e2e-rate-limit-test-subscription"]}
{"level":"info","ts":"2026-07-09T22:48:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"8591502b-6fa7-4b98-b176-6f82a9eaaf7a","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"6d550ae0-6fdb-47c4-a584-ef7ff9f1341b","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"cd7bee28-1d18-4f8b-9ed1-f67a6a63f5ce","MaaSModelRef":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"routeName":"e2e-distinct-2-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-distinct-2-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"1efa719a-34af-4332-92a2-9f41875e7f72","MaaSModelRef":{"name":"e2e-distinct-simulated","namespace":"llm"},"routeName":"e2e-distinct-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-distinct-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-trlp-test-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-trlp-test-simulated","reconcileID":"a121fe37-4ebf-42c1-993e-e1550fcad980","MaaSModelRef":{"name":"e2e-trlp-test-simulated","namespace":"llm"},"routeName":"e2e-trlp-test-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-trlp-test-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"400fe351-b647-4ddb-bc26-69e6f0505b14","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8bbdfe35-cdf2-429c-b727-20aa55b86cdc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"79e2ab0a-037c-4eef-8d22-022f321dbb2d","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"79e2ab0a-037c-4eef-8d22-022f321dbb2d"}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"79e2ab0a-037c-4eef-8d22-022f321dbb2d","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:48:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"5c615db4-d649-4dfe-b697-7e86ef470671","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"5c615db4-d649-4dfe-b697-7e86ef470671"}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"5c615db4-d649-4dfe-b697-7e86ef470671","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:48:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"c0c8daef-e71c-4ab2-9a12-4c218192d09a","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"c0c8daef-e71c-4ab2-9a12-4c218192d09a"}
{"level":"info","ts":"2026-07-09T22:48:16Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"c0c8daef-e71c-4ab2-9a12-4c218192d09a","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:48:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"33d6df5f-5cda-48cf-bffd-8741ca58f161","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"33d6df5f-5cda-48cf-bffd-8741ca58f161","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"027e1827-2ed0-4ec8-98b5-cb96f1d01272","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"027e1827-2ed0-4ec8-98b5-cb96f1d01272","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"9300f88f-e498-4c29-aa66-fa266d1c12de","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:20Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:20Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"ffe58f79-2cc4-4418-92ca-a6e28e1211df","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:48:20Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"ffe58f79-2cc4-4418-92ca-a6e28e1211df"}
{"level":"info","ts":"2026-07-09T22:48:20Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"ffe58f79-2cc4-4418-92ca-a6e28e1211df","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:48:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"de5bf962-38bb-4ab6-a846-5345a2b4e630","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"de5bf962-38bb-4ab6-a846-5345a2b4e630","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"adb41383-b1c2-4a49-bf0c-df9c4162e44c","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"adb41383-b1c2-4a49-bf0c-df9c4162e44c","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"d92ccd0d-5e9a-4beb-845f-20723c9602c8","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"d92ccd0d-5e9a-4beb-845f-20723c9602c8","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"526c2cf4-14a5-45c7-ad20-404bf5002811","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"526c2cf4-14a5-45c7-ad20-404bf5002811","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"bde38708-cb9a-4267-af84-ef23ea3213bb","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T22:48:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"10a4be28-6f02-46ef-959d-8324dd36495b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T22:48:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"2438d2e4-31b2-4767-9491-c6adc0437f99","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"2438d2e4-31b2-4767-9491-c6adc0437f99","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"5bc7b9e8-70c8-49c2-b0af-69277126323e","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"5bc7b9e8-70c8-49c2-b0af-69277126323e","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"e9f8ed34-fce4-4216-8e83-39842b5999f5","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"e9f8ed34-fce4-4216-8e83-39842b5999f5","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:48:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:48:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"1ab3e506-50e5-49f4-81e0-b8e2227a4f44","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"1ab3e506-50e5-49f4-81e0-b8e2227a4f44","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"9077f9ab-1f5f-4f52-838d-1732d563ce5b","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"9077f9ab-1f5f-4f52-838d-1732d563ce5b","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"059713da-3ce9-4d86-8e3a-e948bdc87001","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"059713da-3ce9-4d86-8e3a-e948bdc87001","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:50Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"af395488-f9a9-424c-b1d5-853614b4c154","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"model":"llm/e2e-unconfigured-facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T22:48:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"8cacac74-0c1b-4fb5-a53f-a8890e690cdf","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"c3a15878-b886-4726-9db0-55ba962f4092","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:48:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"ff7be940-559a-4613-83dd-0945c93e8a1a","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"ff7be940-559a-4613-83dd-0945c93e8a1a","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"b4205ff3-9916-45d0-874a-71cc5b881cab","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:55Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"f9635dc5-80fe-4f28-a944-f6dd219a1fe0","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"9fd525fe-6ab1-45d7-8d96-de45c5504912","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"9fd525fe-6ab1-45d7-8d96-de45c5504912"}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"9fd525fe-6ab1-45d7-8d96-de45c5504912","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"c6e7bdaa-ef2f-4abf-8ed7-66ed6a2afab6","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:55Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"c183e33f-c71a-4892-8116-24c1f6e3ab84","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"c183e33f-c71a-4892-8116-24c1f6e3ab84"}
{"level":"info","ts":"2026-07-09T22:48:55Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"c183e33f-c71a-4892-8116-24c1f6e3ab84","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:48:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"c9f806d2-ed43-418e-8510-9285276a5304","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"c9f806d2-ed43-418e-8510-9285276a5304","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"c617773c-13be-402c-a842-99c4fb6d14ed","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"5b5195a0-237a-4a53-8b7b-4eb081bd5608","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:48:57Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:48:57Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"3986f18c-adda-4f2c-9f8a-f72d52784f8a","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-09T22:48:57Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"5b5195a0-237a-4a53-8b7b-4eb081bd5608","name":"e2e-unconfigured-facebook-opt-125m-simulated","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"e2e-unconfigured-facebook-opt-125m-simulated\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:48:57Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"3986f18c-adda-4f2c-9f8a-f72d52784f8a"}
{"level":"info","ts":"2026-07-09T22:48:57Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"3986f18c-adda-4f2c-9f8a-f72d52784f8a","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:48:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"fde4ed17-00a8-48ed-bbfa-6208003ef2a3","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:48:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"fde4ed17-00a8-48ed-bbfa-6208003ef2a3","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"eceb9ab9-19b4-4797-8206-a6c0322e8398","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"eceb9ab9-19b4-4797-8206-a6c0322e8398","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"820271c2-3893-4ca4-9125-3c1a7412ebd3","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"820271c2-3893-4ca4-9125-3c1a7412ebd3","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"ab7c49ca-ad3c-48d5-afd5-dec9e1a5210d","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"ab7c49ca-ad3c-48d5-afd5-dec9e1a5210d","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"dadb716b-2caa-421a-a00d-d278e1544f09","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"dadb716b-2caa-421a-a00d-d278e1544f09","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"3f9925b5-5fe3-4bec-9db1-d3beb9d21ff5","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:11Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"818cdd47-33ae-4e9d-860a-5b1d651a4864","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated","model":"llm/e2e-unconfigured-facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/e2e-models-exempt-test-subscription"]}
{"level":"info","ts":"2026-07-09T22:49:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"b9536809-6dbd-4b80-80b7-055871e3ebc9","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"52e0047d-1766-4d50-b520-d83d466a25a7","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:12Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:49:12Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"785ab58b-5208-4b14-b9b4-a92684bf2a5f","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:49:12Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"785ab58b-5208-4b14-b9b4-a92684bf2a5f"}
{"level":"info","ts":"2026-07-09T22:49:12Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"785ab58b-5208-4b14-b9b4-a92684bf2a5f","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:49:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:49:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:49:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"549082e4-ae02-4e85-87b1-91f7aef317b6","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"549082e4-ae02-4e85-87b1-91f7aef317b6","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:49:16Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"eaa580cf-26f2-4e6b-8da7-f711a140a682","tenant":"models-as-a-service/default-tenant","tenantID":"","subscriptionNamespace":"models-as-a-service","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:49:16Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"error","ts":"2026-07-09T22:49:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"e924e104-babb-4122-8741-11c6a324bad6","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"e924e104-babb-4122-8741-11c6a324bad6","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"3233015d-be9a-4ea3-b336-75364b2d2691","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"3233015d-be9a-4ea3-b336-75364b2d2691","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"a3fa9d86-4aa1-45bb-be45-bd59b969dc5c","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"a3fa9d86-4aa1-45bb-be45-bd59b969dc5c","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:23Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"1fe0ede1-653e-4665-a8a5-44654ee1c666","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-09T22:49:23Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5b5f8018-b3f4-4ae4-92e6-9908d096efc8","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T22:49:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"850d842d-067a-4e4b-870c-ea5c53de1377","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"850d842d-067a-4e4b-870c-ea5c53de1377","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"cbc03495-e8b1-4eaf-b0d2-2268fe26411f","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"cbc03495-e8b1-4eaf-b0d2-2268fe26411f","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"f20c5320-6909-41e8-9c8b-421faa3b1721","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"f20c5320-6909-41e8-9c8b-421faa3b1721","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"81e57395-f571-43ab-805f-09adee2c1a8e","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"81e57395-f571-43ab-805f-09adee2c1a8e","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:38Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"46d97154-2587-4857-9920-1e00c684e512","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"46d97154-2587-4857-9920-1e00c684e512","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"f7debde9-bd79-4821-8754-ea35d93d7224","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"f7debde9-bd79-4821-8754-ea35d93d7224","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:43Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"18beb558-a5dd-44d0-ab0a-d0a64da6802c","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"model":"llm/e2e-unconfigured-facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T22:49:43Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"946c0988-7483-4bba-a9c6-e2d87bbfecd9","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:43Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"21b3ac85-1ef4-4336-bff1-9e1ac6c29287","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:49:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:49:44Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"8477fa7e-fcfd-4ddc-87fa-86dd79ce12be","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:44Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"8477fa7e-fcfd-4ddc-87fa-86dd79ce12be","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:47Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"d38128e3-5ede-49b7-9ab5-87aa80b6222d","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:47Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"d38128e3-5ede-49b7-9ab5-87aa80b6222d","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ab24bfa2-70eb-44e1-95e9-b7b52ec97f99","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e7b4a683-2b37-40f8-8a9a-314069a59c03","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c7c88d68-7006-4d5b-aeac-00680a34234d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:49:50Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"5580f4af-1d44-48fb-ad82-76b50c9ce8c9","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"5580f4af-1d44-48fb-ad82-76b50c9ce8c9","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"42b0c605-01c1-4806-b1bb-b7062d7e4376","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"3b24d2cf-4f17-4def-8625-18e88ad77d1c","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:51Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"fa1d276f-67bb-4f7d-88c3-e06eba7d8c00","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"fa1d276f-67bb-4f7d-88c3-e06eba7d8c00"}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"fa1d276f-67bb-4f7d-88c3-e06eba7d8c00","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"7ae39314-42f0-4821-8cda-c1a763a5ff6e","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"a0da316e-4140-4457-8ccb-235e36024dc1","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"d59e11f6-5201-4626-bf90-aa65d57b346e","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:49:53Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"78cc736a-2dfd-4cf5-abb0-1e45ddc5e293","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"78cc736a-2dfd-4cf5-abb0-1e45ddc5e293","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:49:55Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"5d9f3af4-bec9-4883-bfc6-3134c7a99eb0","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:56Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"4c5782c8-472f-4580-8939-f279ce7dd9e3","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-09T22:49:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"c50eda5a-cd8e-4ef2-9aff-43f94435ca48","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:49:56Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"899a6484-feea-4c1a-b89d-8c288c7215c1","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:49:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"7f5d66bf-9c2e-4fc2-839c-d1f1035918b9","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"7f5d66bf-9c2e-4fc2-839c-d1f1035918b9","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"26421b3e-b8c9-49ef-93e5-12f0b6bb05e3","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:49:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"26421b3e-b8c9-49ef-93e5-12f0b6bb05e3","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"87f5538c-d6f5-4940-9563-5ccce923981d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"29bc02b4-4b7b-4d01-95c2-c2be6a43662c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:00Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:50:00Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"51b122cb-5788-4578-8e94-38ea52e9e7a5","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:50:00Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"51b122cb-5788-4578-8e94-38ea52e9e7a5"}
{"level":"info","ts":"2026-07-09T22:50:00Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"51b122cb-5788-4578-8e94-38ea52e9e7a5","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:50:00Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"270a3c73-d80d-4df6-8518-233559bb0a42","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:50:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"01d40041-4abb-4e30-b7da-d9204ccec22c","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"01d40041-4abb-4e30-b7da-d9204ccec22c","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:04Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"7cab54a2-de3e-4cba-a408-74fdc2642efe","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-09T22:50:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"65ddd805-175b-41eb-b666-4cfc2eb9a05c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"152fa06e-da3f-46d2-89b3-66ee9c7aa5d8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:50:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"6cf0a272-414f-4e4e-8d89-a61eb7e7210e","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"6cf0a272-414f-4e4e-8d89-a61eb7e7210e","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"ed171297-7b11-48e0-9380-2913d9e27971","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"ed171297-7b11-48e0-9380-2913d9e27971","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"94748824-6eec-4133-84f2-a9e33b08c7f0","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"94748824-6eec-4133-84f2-a9e33b08c7f0","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"cfaaa3a7-d769-4d5c-8422-a17ccb540767","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e1177832-b323-415b-96bf-784bd3a008e5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:50:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:50:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"6780fe3a-0ce5-4806-b3e4-8d244cc9f07b","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"6780fe3a-0ce5-4806-b3e4-8d244cc9f07b","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:16Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"5a16c146-060d-476e-a774-1638c88cad9f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c5dc9c71-838a-4548-b03d-3bb4fe8b99fe","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"07756d0c-5a4f-4571-b08e-7f90a0c6ce34","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:50:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"f2f82f5c-55ac-4827-a092-c9a0e51cfa87","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"f2f82f5c-55ac-4827-a092-c9a0e51cfa87","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"b408f999-8383-44ab-ad4a-967037ebba4d","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"b408f999-8383-44ab-ad4a-967037ebba4d","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e389b964-fc2c-4d86-8705-7608fc10e66d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:50:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"08428c4c-d6b1-4e1b-b297-352a0956a79f","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"08428c4c-d6b1-4e1b-b297-352a0956a79f","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:25Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7ceb7682-68d7-4be2-951b-d4fe1c530265","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:50:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"bcf87213-9bcd-4501-86df-15de22d74ef4","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"bcf87213-9bcd-4501-86df-15de22d74ef4","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"569a1758-4c4c-4567-972e-5f3353d4771a","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"569a1758-4c4c-4567-972e-5f3353d4771a","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:29Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"319167ec-55ac-40ef-953d-d8e571e25f28","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:50:29Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:50:29Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"dbe0be63-2b44-44f7-aa1b-b23f6287a60f","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:50:29Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"dbe0be63-2b44-44f7-aa1b-b23f6287a60f"}
{"level":"info","ts":"2026-07-09T22:50:29Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"dbe0be63-2b44-44f7-aa1b-b23f6287a60f","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:50:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"29a2e96f-aba6-4225-bb25-6ee1719911e6","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"29a2e96f-aba6-4225-bb25-6ee1719911e6","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"0e4f7d96-1194-47d5-a33e-c1bff38a57c0","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"0e4f7d96-1194-47d5-a33e-c1bff38a57c0","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:38Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"010842ba-f80a-457d-8daa-6719899fb04b","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"010842ba-f80a-457d-8daa-6719899fb04b","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"c2bcac7d-2204-4d86-a4f8-9fb303ef3e08","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"c2bcac7d-2204-4d86-a4f8-9fb303ef3e08","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:50:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:50:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:50:44Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"08fb1d17-eb4a-482a-97fb-cbef386ebc1b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:44Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"08fb1d17-eb4a-482a-97fb-cbef386ebc1b","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:47Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"ef462156-1c80-4eae-85bb-313d75e362e6","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:47Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"ef462156-1c80-4eae-85bb-313d75e362e6","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:50Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"466b00fd-e13a-4446-8302-dc8e30ae54b2","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"466b00fd-e13a-4446-8302-dc8e30ae54b2","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:53Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"49530da4-f8cf-40bd-b94f-5244e7b3d03b","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"49530da4-f8cf-40bd-b94f-5244e7b3d03b","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"9355a46e-ead3-4f9c-95a4-151e01e53e1c","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"9355a46e-ead3-4f9c-95a4-151e01e53e1c","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"7c4b5560-758a-474a-b598-0ba18f4bae9f","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:50:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"7c4b5560-758a-474a-b598-0ba18f4bae9f","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"df224fea-dbf9-4374-8dff-2cae9828a091","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-6360a52c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-6360a52c","reconcileID":"df224fea-dbf9-4374-8dff-2cae9828a091","error":"failed to revoke API keys for subscription e2e-filter-sub-a-6360a52c after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"cd732484-8d91-4a8c-9ae0-5fc615b3a79a","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"cd732484-8d91-4a8c-9ae0-5fc615b3a79a","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"3f515a78-ff54-4ad5-84c1-8005c30a0c4c","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"3f515a78-ff54-4ad5-84c1-8005c30a0c4c","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:51:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"fb873605-dc5e-409c-8989-dc0350f31202","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:51:09Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"e93b6732-c82a-4c15-bce8-9e440b65fe53","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:51:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"c9c9ebb2-2851-4845-a5c5-0fb3bfb58c4e","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"c9c9ebb2-2851-4845-a5c5-0fb3bfb58c4e","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:51:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:51:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:51:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"f4ddf4b3-87ae-4eb0-b21c-a324ea895d2b","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"f4ddf4b3-87ae-4eb0-b21c-a324ea895d2b","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"37f1c2e3-a690-4d01-82f4-7296e73b3018","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"37f1c2e3-a690-4d01-82f4-7296e73b3018","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"d60932eb-a525-499c-935b-39d67f9f932a","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"d60932eb-a525-499c-935b-39d67f9f932a","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"6c9fadde-0d4d-48ca-84b5-7b105885c9ee","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"6c9fadde-0d4d-48ca-84b5-7b105885c9ee","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"cfb8e9f6-600f-4911-a9f7-4add63af9110","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"cfb8e9f6-600f-4911-a9f7-4add63af9110","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:51:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"337f2ff9-6c23-46aa-9375-51dfdbc89d19","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:51:26Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"51be58d7-509e-4e1b-9807-dfbdf183a363","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-ordering-sub","models-as-a-service/premium-simulator-subscription"]}
{"level":"info","ts":"2026-07-09T22:51:26Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"799e60da-af3a-4bb1-bad9-b52e012a9b23","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:51:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"73efdf2f-4aec-4588-8a13-41bc8c57d935","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:51:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"908c3ece-2031-47c2-b276-6568c98c58ca","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:51:28Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:51:28Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"a82147a4-da3b-48d1-92cb-e80cc5748f39","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:51:28Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"a82147a4-da3b-48d1-92cb-e80cc5748f39"}
{"level":"info","ts":"2026-07-09T22:51:28Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"a82147a4-da3b-48d1-92cb-e80cc5748f39","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-09T22:51:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"2fed2e87-ad58-46c6-98b5-be1569a79f84","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:51:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"8d25d9db-95f2-4196-b3ff-ac2749f92483","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"8d25d9db-95f2-4196-b3ff-ac2749f92483","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"c0f24d5e-e0c0-47d2-9ff5-601246d66996","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"c0f24d5e-e0c0-47d2-9ff5-601246d66996","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:51:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"a39c2bf2-034d-480e-ab21-c76421520bec","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:51:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"980d5c9f-aaea-43d3-94a0-a6a3fad3524d","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"980d5c9f-aaea-43d3-94a0-a6a3fad3524d","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:38Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"21c308f1-e485-4d7e-bfb9-91e86b209a7d","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"21c308f1-e485-4d7e-bfb9-91e86b209a7d","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"3b29d099-35f7-4d9f-870b-4d350c8344e4","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"3b29d099-35f7-4d9f-870b-4d350c8344e4","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:51:41Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"4c1e3266-94b1-4fc5-b8f4-d17978b20a89","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1,"subscriptions":["models-as-a-service/premium-simulator-subscription"]}
{"level":"info","ts":"2026-07-09T22:51:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:51:43Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-09T22:51:44Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"39f7f616-1cde-4ee0-937e-259400f4174c","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:44Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"39f7f616-1cde-4ee0-937e-259400f4174c","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:47Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"8625583e-a980-4690-b4c9-6576fed0a0bc","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:47Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"8625583e-a980-4690-b4c9-6576fed0a0bc","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:50Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"2e60bd31-6f40-4638-b8a6-ff4494b823b4","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"2e60bd31-6f40-4638-b8a6-ff4494b823b4","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:53Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"525f01b2-47f7-47c5-905c-ec6ba1564680","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"525f01b2-47f7-47c5-905c-ec6ba1564680","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:51:53Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"526313b4-1cdb-4f00-86b1-a02621617b0e","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-09T22:51:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"b2e40f83-eace-442a-998c-ea350b2284e0","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"b2e40f83-eace-442a-998c-ea350b2284e0","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"749e364e-b37c-4ae6-a2fd-5d343436c073","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:51:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f49cfd0a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f49cfd0a","reconcileID":"749e364e-b37c-4ae6-a2fd-5d343436c073","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f49cfd0a after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"1f736b24-3a22-41bc-9dbd-2ea775bd8b18","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-73f2bb97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-73f2bb97","reconcileID":"1f736b24-3a22-41bc-9dbd-2ea775bd8b18","error":"failed to revoke API keys for subscription e2e-filter-sub-b-73f2bb97 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:52:05Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"038af126-34a8-45ad-9e33-a0062fabf010","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-09T22:52:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"f86a477e-3e63-4b3c-8ad1-b8b99c7bf6ba","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:52:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"8635a275-a756-4d82-996d-5556e49b09c7","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:52:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"555d8029-1efd-4192-aa39-6ee6d0e48b5f","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"555d8029-1efd-4192-aa39-6ee6d0e48b5f","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"9031f2a6-f281-4229-97c4-77c5a0c2cb1e","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"9031f2a6-f281-4229-97c4-77c5a0c2cb1e","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"e0a8f190-ee4c-48a1-a3a1-b405cabef39b","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"e0a8f190-ee4c-48a1-a3a1-b405cabef39b","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:52:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:52:13Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-09T22:52:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d6fedb5b-6669-4b90-a5d5-cf2985b6f340","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-09T22:52:14Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-09T22:52:14Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"19fe8579-273e-4be3-9011-9fa63d9ca9e0","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-09T22:52:14Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"19fe8579-273e-4be3-9011-9fa63d9ca9e0"}
{"level":"info","ts":"2026-07-09T22:52:14Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"19fe8579-273e-4be3-9011-9fa63d9ca9e0","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-09T22:52:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"f17c6984-a788-4640-b0b9-9a1b0441dcea","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"f17c6984-a788-4640-b0b9-9a1b0441dcea","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"832dc3f6-19d5-4321-b319-92e26f6fbad7","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-9f62be12","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-9f62be12","reconcileID":"832dc3f6-19d5-4321-b319-92e26f6fbad7","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-9f62be12 after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-09T22:52:19Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0d91f2c7-5610-4115-b0e3-1eec1d0165f1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-09T22:52:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"611deadb-b7f7-4ebc-83ba-dcc25da5309c","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"611deadb-b7f7-4ebc-83ba-dcc25da5309c","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"447e2fb6-afe9-42dc-ac42-8ac42f1f509b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"447e2fb6-afe9-42dc-ac42-8ac42f1f509b","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"3cee6f9d-f3b9-4881-8735-cbc08492fbce","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-09T22:52:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"3cee6f9d-f3b9-4881-8735-cbc08492fbce","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP 403: {\"details\":\"This endpoint is only accessible by the maas-controller\",\"error\":\"Insufficient permissions\"}","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}