{"level":"info","ts":"2026-07-03T20:51:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"ca17e9af-afa3-44d3-873a-8fabeb8bd80c","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d8322bf7-f547-4e85-8b70-459fbdfca046","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c6996481-3099-45d3-a9cd-fe2e6d1a6532","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:02Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"35e986e3-6761-4f07-8978-0008c36a6ef5","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"35e986e3-6761-4f07-8978-0008c36a6ef5"}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"35e986e3-6761-4f07-8978-0008c36a6ef5","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"816ced14-722a-4540-8e1b-e4c307cde227","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:02Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"5ffbfd9b-eb70-41a1-9454-516e1219e475","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"5ffbfd9b-eb70-41a1-9454-516e1219e475"}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"5ffbfd9b-eb70-41a1-9454-516e1219e475","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"83e17699-d9b5-444a-8d18-073e149a4dfa","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"aa57618e-9186-40e4-9419-5d01ce2a78b2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:51:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"7432c1bb-e06e-40f3-976e-f89f3e954898","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"7432c1bb-e06e-40f3-976e-f89f3e954898","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"84d88152-9bc6-454b-b6a7-d7df52919413","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"84d88152-9bc6-454b-b6a7-d7df52919413","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"098f75b8-c009-47f1-97dc-ebda58df46cf","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"098f75b8-c009-47f1-97dc-ebda58df46cf","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"354e26a0-8f33-41c5-9f61-83004157ac54","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"354e26a0-8f33-41c5-9f61-83004157ac54","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"4203844f-3138-4eff-983e-f6acb944d4db","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"4203844f-3138-4eff-983e-f6acb944d4db","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"79ed004f-6b64-429a-b90e-877b04c69df0","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"79ed004f-6b64-429a-b90e-877b04c69df0","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:51:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1ec36f91-7ee9-45ee-b50d-495baf37f41f","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:17Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"caa773ab-d0dc-45da-817a-256fe1797003","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-neg-ghost-sub-f30784b7","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-03T20:51:17Z","msg":"model/route not found during cleanup, deleting TokenRateLimitPolicy via labels","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"caa773ab-d0dc-45da-817a-256fe1797003","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-f30784b7","error":"MaaSModelRef not found: llm/nonexistent-model-f30784b7"}
{"level":"info","ts":"2026-07-03T20:51:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"74c08719-da87-4bc7-aded-4609e246ae77","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:19Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"85f82894-db9b-4011-98cf-93481a7e0203","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:51:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"65c4645d-2b25-4a3c-ab9e-f299a2402305","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"65c4645d-2b25-4a3c-ab9e-f299a2402305","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"648e80d8-9935-47e9-81a8-5772688a402a","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"648e80d8-9935-47e9-81a8-5772688a402a","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:51:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:51:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:51:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"2c5a008d-0813-42f7-bf63-42127e29da46","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"2c5a008d-0813-42f7-bf63-42127e29da46","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"ced186ce-3be2-4c4f-b199-83ac6c41af98","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"ced186ce-3be2-4c4f-b199-83ac6c41af98","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"4440484b-2f59-42ea-8426-4054d02f2dc7","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"4440484b-2f59-42ea-8426-4054d02f2dc7","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:51:32Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"b729466f-ca6d-4b0b-9269-af41594ebed7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"error","ts":"2026-07-03T20:51:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"3bcf4467-2da6-42a2-81c7-83967bfa91ea","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"3bcf4467-2da6-42a2-81c7-83967bfa91ea","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:38Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e576d7ae-d70e-46d6-ac2d-df8974e209fa","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e576d7ae-d70e-46d6-ac2d-df8974e209fa","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"bff822ac-43e8-4081-9a4f-71d4d723e271","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"bff822ac-43e8-4081-9a4f-71d4d723e271","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:51:41Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"08bdddc7-63f0-4738-b9d0-eba593450207","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T20:51:45Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"0a922e40-c3da-489a-a1e9-b283fe966dcf","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:45Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"0a922e40-c3da-489a-a1e9-b283fe966dcf","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:48Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"91cd1f08-91af-48ca-9210-59366613a3b2","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:48Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"91cd1f08-91af-48ca-9210-59366613a3b2","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:51:49Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-f30784b7","reconcileID":"b5655001-c5df-4217-a763-2a022a6c1f92","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-f30784b7","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T20:51:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b5a34ed8-6401-4e80-a854-6140e934490c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fcbe20b1-5a21-4095-b6c5-1646752a315b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c77188d0-5d61-406d-97d8-2ac3bc6fc15b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a6b93410-6da4-46fa-97fb-bb5ff011f64b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"3657682e-6271-4263-af13-038d8309c805","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"3657682e-6271-4263-af13-038d8309c805"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"3657682e-6271-4263-af13-038d8309c805","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"model not found, cleaning up generated AuthPolicy","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"3657682e-6271-4263-af13-038d8309c805","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-82358209"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"213f6f78-255b-4b9e-9d3e-82c761b4ac45","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:50Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"499dd9b4-9627-40aa-97c5-d560eafe750c","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"499dd9b4-9627-40aa-97c5-d560eafe750c"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"499dd9b4-9627-40aa-97c5-d560eafe750c","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"model not found, cleaning up generated AuthPolicy","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"499dd9b4-9627-40aa-97c5-d560eafe750c","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-82358209"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"c6929a99-0572-4719-96c0-3478a3229ab5","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bd027d0a-eaa5-486a-abc7-31f3f9c99173","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-82358209","reconcileID":"c6929a99-0572-4719-96c0-3478a3229ab5","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-82358209","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-82358209"}
{"level":"info","ts":"2026-07-03T20:51:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6f3b4f5c-87bf-459d-b8e3-bed6367f8970","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:51:51Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"8c846e78-6af3-465c-889a-18b17c337c1c","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:51Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"8c846e78-6af3-465c-889a-18b17c337c1c","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:51:51Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"469cf05e-ff76-4e1d-afd7-5a2decf962cf","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T20:51:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:51:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:51:54Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"7b518139-054f-4c45-bc4d-33af53beb802","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:54Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"7b518139-054f-4c45-bc4d-33af53beb802","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:57Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"a17f68b7-d8b4-4da6-9ead-9ad4e51b5258","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:51:57Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"a17f68b7-d8b4-4da6-9ead-9ad4e51b5258","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:00Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e784fa34-214b-4cda-af99-ee627ec3b7e2","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:00Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e784fa34-214b-4cda-af99-ee627ec3b7e2","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:03Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"989828dd-8de0-44fd-992c-7bcbb6bbfc33","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:03Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"989828dd-8de0-44fd-992c-7bcbb6bbfc33","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:52:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"89a651bd-e77b-4f06-a5e7-6b8cfcc081ce","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"8c42a378-b717-4360-b2bb-b4979958778c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:52:06Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"0908192e-10e3-4d52-803f-2280320454b3","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:06Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"0908192e-10e3-4d52-803f-2280320454b3","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:09Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"8bfb35a9-e757-401f-b331-10dd80df2fa4","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:09Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"8bfb35a9-e757-401f-b331-10dd80df2fa4","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:12Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"5e081dd6-c992-43d0-9b35-666ac7043862","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:12Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"5e081dd6-c992-43d0-9b35-666ac7043862","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:15Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"f0e21b60-56e0-4bad-bac7-d4192752a1b4","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:15Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"f0e21b60-56e0-4bad-bac7-d4192752a1b4","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:18Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"9038793c-b718-4e9c-b553-e79297557c9d","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:18Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"9038793c-b718-4e9c-b553-e79297557c9d","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:21Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"1f0e371e-f6f9-44f2-9baf-ccc6d7b22b75","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:21Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"1f0e371e-f6f9-44f2-9baf-ccc6d7b22b75","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:52:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d507a999-387d-4208-9249-1de8a8d33acc","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:21Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"db87132a-a051-42cc-9cc5-4e3eaa95892e","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-apikey-sub-binding-eb9fd595","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-03T20:52:21Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d8b6e500-f5a2-42f1-bcb1-eb8e055e4562","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ec4e43a9-874b-4ad5-899e-9d42a1ca25b2","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:52:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:52:24Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"d53b7df1-51fe-42cc-8160-a81f09abf9ea","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:24Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"d53b7df1-51fe-42cc-8160-a81f09abf9ea","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:27Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"7841ae80-d7f9-4afb-8cd0-db1a23d18e30","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:27Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"7841ae80-d7f9-4afb-8cd0-db1a23d18e30","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:30Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"da2aa116-1797-4c55-9a81-36a8cdd1a627","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:30Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"da2aa116-1797-4c55-9a81-36a8cdd1a627","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:33Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"85d5f96d-0666-4ee8-8d4c-bbb637eef31a","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:33Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"85d5f96d-0666-4ee8-8d4c-bbb637eef31a","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:36Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"18784797-de60-4fa1-945a-fe435b039107","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:36Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"18784797-de60-4fa1-945a-fe435b039107","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:39Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"166db72c-4189-4ec5-9501-3b45f9546831","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:39Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"166db72c-4189-4ec5-9501-3b45f9546831","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:52:39Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"cfe25ac2-5fff-41e4-ac4a-4e3ed1637d1a","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"error","ts":"2026-07-03T20:52:42Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"07ad146b-e683-42b5-b7ba-de187f384b3d","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:42Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"07ad146b-e683-42b5-b7ba-de187f384b3d","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:52:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a93fd9f7-3c60-498f-9c75-5ea1220a9e50","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T20:52:45Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"dcb44c29-0fab-4874-837c-f942253d0cd5","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:45Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"dcb44c29-0fab-4874-837c-f942253d0cd5","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:48Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"23abef0c-50b2-4589-bbea-c83c93283cb4","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:48Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"23abef0c-50b2-4589-bbea-c83c93283cb4","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:51Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"0f2e0e89-b73c-4329-b029-77d0525248a8","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:51Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"0f2e0e89-b73c-4329-b029-77d0525248a8","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:52:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"e9a4c6c7-e494-4936-a53a-53fa4adf95d7","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"e3ebdb2b-ed52-4537-a8d7-fe4e18600a98","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:53Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:52:53Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"b87c60c8-8143-4ded-9d2c-fbbd5cbe2ab7","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:52:53Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"b87c60c8-8143-4ded-9d2c-fbbd5cbe2ab7"}
{"level":"info","ts":"2026-07-03T20:52:53Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"b87c60c8-8143-4ded-9d2c-fbbd5cbe2ab7","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:52:53Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"57a12322-649e-4f61-82cc-c4921a74a1b4","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:52:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:52:54Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"5c1a1f3b-9b76-4c3d-8411-2dbca1725270","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:54Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"5c1a1f3b-9b76-4c3d-8411-2dbca1725270","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:52:57Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"da9a89e4-2812-4f7f-b9b7-a373eae22048","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-03T20:52:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"ddd1aa30-0237-4df8-b570-51771e11da5e","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:52:57Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"44fcf13f-8467-49dd-8199-bb71170ab777","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:52:57Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"4a8361ac-4855-4478-a458-7f441f010bc2","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:52:57Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"4a8361ac-4855-4478-a458-7f441f010bc2","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:00Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"8c466a8d-1b10-4902-92d5-28a84335607f","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:00Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"8c466a8d-1b10-4902-92d5-28a84335607f","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:00Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"dfbd3350-dbd2-4189-a0ea-fe51559fa68f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"c2b404e3-c122-423e-ab29-c1b090a6cc83","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:01Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"a2e7ec91-3c09-45b1-bc06-cc8a6698e5f5","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"b0bfa524-e932-4128-b496-574f545aefe7","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"b0bfa524-e932-4128-b496-574f545aefe7"}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"b0bfa524-e932-4128-b496-574f545aefe7","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"81e26abe-274d-45f3-8e72-db71401bc4cd","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:01Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"9ae80c7e-074a-4b26-8b80-126574359298","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"9ae80c7e-074a-4b26-8b80-126574359298"}
{"level":"info","ts":"2026-07-03T20:53:01Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"9ae80c7e-074a-4b26-8b80-126574359298","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:53:03Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"870a4d99-a0f5-489d-95aa-8ccbe7fdece2","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:03Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"870a4d99-a0f5-489d-95aa-8ccbe7fdece2","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"2fa0aa69-558b-4bc7-bc2f-f5da060df818","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"d0a8fcec-e356-4ec4-838e-5c16076bd8ad","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:05Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:05Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"ef52c3c8-5882-43b4-a1d2-c8f1cc842e9c","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-03T20:53:05Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"d0a8fcec-e356-4ec4-838e-5c16076bd8ad","name":"e2e-unconfigured-facebook-opt-125m-simulated","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"e2e-unconfigured-facebook-opt-125m-simulated\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:05Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"ef52c3c8-5882-43b4-a1d2-c8f1cc842e9c"}
{"level":"info","ts":"2026-07-03T20:53:05Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"ef52c3c8-5882-43b4-a1d2-c8f1cc842e9c","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:53:06Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"77a1efae-9dbc-4d64-9b97-3f36f424e801","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:06Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"77a1efae-9dbc-4d64-9b97-3f36f424e801","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:06Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"bbd601f6-e982-4d5b-91c9-59016e7e6d9e","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T20:53:09Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e74f15d6-dfcc-428e-8b0d-e84169ecfaf4","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:09Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"e74f15d6-dfcc-428e-8b0d-e84169ecfaf4","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:12Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"5ae55ae6-33f9-4341-b186-fd10406f4a8f","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:12Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"5ae55ae6-33f9-4341-b186-fd10406f4a8f","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:15Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"534a9e84-1138-4a47-8035-50e00ba5bb90","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:15Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"534a9e84-1138-4a47-8035-50e00ba5bb90","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:18Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"14cf9bfa-75b2-4c8b-8bbb-8efae6af4712","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:18Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"14cf9bfa-75b2-4c8b-8bbb-8efae6af4712","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:21Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"f049e5af-d0f1-48e5-bf42-c39258aa0514","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:21Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"f049e5af-d0f1-48e5-bf42-c39258aa0514","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:53:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:53:24Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"3ea4e3ae-7c39-4e82-9dee-836d1a1f9f7f","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:24Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"3ea4e3ae-7c39-4e82-9dee-836d1a1f9f7f","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"8b8614d6-e5c6-4dc4-ad48-c85e6d62d7d1","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:24Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b06da47f-a040-413d-b1e8-9beb27ceef58","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated","model":"e2e-unconfigured-facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/e2e-rate-limit-test-subscription"]}
{"level":"info","ts":"2026-07-03T20:53:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"4b0f2df3-dee0-483e-979e-b03b370a41d2","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"fb4988bf-c9dd-459e-a8d2-6ed36af18917","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"874d9366-ff74-4334-ac92-73b0439b45c1","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"977e3ee0-0c8e-4049-bfa0-7fdb9d9ea43e","MaaSModelRef":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"routeName":"e2e-distinct-2-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-distinct-2-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"56bd70ba-5f8f-405b-b6b0-1499f08b1a2c","MaaSModelRef":{"name":"e2e-distinct-simulated","namespace":"llm"},"routeName":"e2e-distinct-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-distinct-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-trlp-test-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-trlp-test-simulated","reconcileID":"b7bed65b-f4af-4819-b235-becc793558dc","MaaSModelRef":{"name":"e2e-trlp-test-simulated","namespace":"llm"},"routeName":"e2e-trlp-test-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-trlp-test-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"62bda5e1-1d32-441b-b6df-033de0b97fa1","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"f5d7d237-7864-4c12-98b3-8d3873ad2cac","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"f5d7d237-7864-4c12-98b3-8d3873ad2cac"}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"f5d7d237-7864-4c12-98b3-8d3873ad2cac","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:53:27Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"95e65099-881b-48f8-8e3f-f9ae8830a9cf","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"95e65099-881b-48f8-8e3f-f9ae8830a9cf"}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"95e65099-881b-48f8-8e3f-f9ae8830a9cf","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:53:27Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"966c24d4-e0df-4ee5-a5e9-31c13608f110","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"966c24d4-e0df-4ee5-a5e9-31c13608f110"}
{"level":"info","ts":"2026-07-03T20:53:27Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"966c24d4-e0df-4ee5-a5e9-31c13608f110","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:53:27Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"c3bd675f-b5a4-4212-a2fd-34705bd3843c","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:27Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"c3bd675f-b5a4-4212-a2fd-34705bd3843c","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:30Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"ba482b23-3e8f-4a0a-bac7-f89c77bef78a","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:30Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"ba482b23-3e8f-4a0a-bac7-f89c77bef78a","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"ed4e7409-bf33-43ae-901e-18b2bb1b4a7b","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:53:31Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:53:31Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"2faf6f2d-56cc-4b7c-aa6e-cf1f5e781ed6","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:53:31Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"2faf6f2d-56cc-4b7c-aa6e-cf1f5e781ed6"}
{"level":"info","ts":"2026-07-03T20:53:31Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"2faf6f2d-56cc-4b7c-aa6e-cf1f5e781ed6","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:53:33Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"406f41a6-1f5d-49e9-ae44-ce45de4c8d0b","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:33Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"406f41a6-1f5d-49e9-ae44-ce45de4c8d0b","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:36Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"1bc0c5d3-32f3-4def-99b2-0fca290106ed","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:36Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"1bc0c5d3-32f3-4def-99b2-0fca290106ed","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:39Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"7af0ff08-079e-45be-94c8-b0121ca699a4","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:39Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"7af0ff08-079e-45be-94c8-b0121ca699a4","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:42Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"9ba74ad5-d39b-4f0f-be37-23d5a7b1073a","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:42Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"9ba74ad5-d39b-4f0f-be37-23d5a7b1073a","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"b3d039e7-4122-4105-a9f8-bef7b0a38fa2","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T20:53:42Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c71ec41f-d652-4d8d-a8af-7f19b9f8f9e3","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T20:53:45Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"47c90f64-6c65-42a9-b20a-3a8a01ab36de","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:45Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"47c90f64-6c65-42a9-b20a-3a8a01ab36de","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:48Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"991c0c87-ae7a-4ff6-bce3-7a6531f506d3","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:48Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"991c0c87-ae7a-4ff6-bce3-7a6531f506d3","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:51Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"a1e4b85d-0ba6-463e-96f7-86dc044a1cf7","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:51Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"a1e4b85d-0ba6-463e-96f7-86dc044a1cf7","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:53:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:53:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:53:54Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"145f73a9-4128-4c93-85ac-76af3db66e1a","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:54Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"145f73a9-4128-4c93-85ac-76af3db66e1a","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:57Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"14ae08ab-c4d0-4e20-8baf-21774be35e15","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:53:57Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"14ae08ab-c4d0-4e20-8baf-21774be35e15","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:00Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b8512669-30d1-43d8-bb99-58c8603cbe9a","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:00Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b8512669-30d1-43d8-bb99-58c8603cbe9a","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:02Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"88b325f0-3b78-4b8d-8dd9-6126f9819984","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"model":"llm/e2e-unconfigured-facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T20:54:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"f45474c3-d5f0-4008-9ac4-ee0db5ca5cbc","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"67e82043-3a6a-4d5a-8453-08241b03e916","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:54:03Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"a78737a5-20a1-478f-a48d-b1432d911023","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:03Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"a78737a5-20a1-478f-a48d-b1432d911023","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"0593f515-ae99-479d-9506-2c4426bdc0e6","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"349717e9-8ffe-4f8a-aa09-965c38d1afc8","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:06Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a42c8bbb-e459-4341-9ff1-e2fa4d6085c2","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a42c8bbb-e459-4341-9ff1-e2fa4d6085c2"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a42c8bbb-e459-4341-9ff1-e2fa4d6085c2","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"282eb27a-a729-4353-9280-c4b25956a71f","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:06Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"622222a9-5060-408b-aef5-3d94a284c138","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"622222a9-5060-408b-aef5-3d94a284c138"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"622222a9-5060-408b-aef5-3d94a284c138","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"e43c70d9-f8f6-4549-9f54-5e7401c9411e","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"9805ced6-e5ee-410e-a504-af506c4042f6","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:06Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"541f3eef-0b8d-48f6-aad2-db9c5643d2a2","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-03T20:54:06Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"9805ced6-e5ee-410e-a504-af506c4042f6","name":"e2e-unconfigured-facebook-opt-125m-simulated","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"e2e-unconfigured-facebook-opt-125m-simulated\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"541f3eef-0b8d-48f6-aad2-db9c5643d2a2"}
{"level":"info","ts":"2026-07-03T20:54:06Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"541f3eef-0b8d-48f6-aad2-db9c5643d2a2","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:54:06Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"7503a7a4-0fde-4c53-892b-49014b2473df","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:06Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"7503a7a4-0fde-4c53-892b-49014b2473df","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"c9b534b3-45bb-429b-8cb7-2a6a281d6d00","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"c9b534b3-45bb-429b-8cb7-2a6a281d6d00","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"da6c9cc1-a1ff-49fb-9a8c-406b3363d3bd","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"da6c9cc1-a1ff-49fb-9a8c-406b3363d3bd","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"96aa5395-f345-44ee-a3a9-ad247d15b117","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"96aa5395-f345-44ee-a3a9-ad247d15b117","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"3ed275ed-4a0a-4e06-ae49-5fb5daf3a40d","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"3ed275ed-4a0a-4e06-ae49-5fb5daf3a40d","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"4e01ca23-8bfb-489d-afdd-13276b3ea7ff","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"4e01ca23-8bfb-489d-afdd-13276b3ea7ff","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"c5956a4b-4619-455a-b333-922f62bd4e2a","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:22Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"cecc943e-7f2c-480c-b1bc-4ba6a063ffff","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated","model":"llm/e2e-unconfigured-facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/e2e-models-exempt-test-subscription"]}
{"level":"info","ts":"2026-07-03T20:54:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"dd7e6d30-e1be-4e50-a760-78ddb9ea54c3","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:54:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:54:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"6bcb3a3d-38b4-46d4-ba61-032cfb386558","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:24Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:24Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"5c6f7d87-a732-4860-9d5b-24d573c9d5f3","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:54:24Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"5c6f7d87-a732-4860-9d5b-24d573c9d5f3"}
{"level":"info","ts":"2026-07-03T20:54:24Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"5c6f7d87-a732-4860-9d5b-24d573c9d5f3","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:54:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"d4d54191-743c-425a-a6d8-9da387bfb247","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"d4d54191-743c-425a-a6d8-9da387bfb247","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:26Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"21a26bb0-7dbd-43b2-a377-d1e684a074f2","tenant":"models-as-a-service/default-tenant","tenantID":"","subscriptionNamespace":"models-as-a-service","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:54:26Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"error","ts":"2026-07-03T20:54:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b4cffa40-259e-4003-b376-d2c60e150824","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b4cffa40-259e-4003-b376-d2c60e150824","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"a6f61106-1865-41be-be2a-96a612f64bee","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"a6f61106-1865-41be-be2a-96a612f64bee","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"beddcc8e-49ed-4512-85ec-54d281a9523d","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"beddcc8e-49ed-4512-85ec-54d281a9523d","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"f8f355c6-de7e-4198-9053-e1f81b28ec77","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T20:54:34Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"53f9aecb-ea01-4c87-bc9b-bca08069b9cc","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T20:54:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"f59e8026-39e5-4a31-bcb5-5593a2aecf82","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"f59e8026-39e5-4a31-bcb5-5593a2aecf82","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"f0794205-1886-4c30-9472-6fd45e6b28c8","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"f0794205-1886-4c30-9472-6fd45e6b28c8","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"74a1d54a-b4ea-4dc7-aaa2-1a99493d95d0","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"74a1d54a-b4ea-4dc7-aaa2-1a99493d95d0","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"92393179-823d-4208-8ccd-9244a1900a25","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"92393179-823d-4208-8ccd-9244a1900a25","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"8a4d01ed-5a40-47cb-88d9-e3921694927f","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"8a4d01ed-5a40-47cb-88d9-e3921694927f","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"3ddff325-1fbd-45c8-a1b1-4cea4fe2c37c","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"3ddff325-1fbd-45c8-a1b1-4cea4fe2c37c","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:54:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:54:54Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"e829cda9-7812-44e1-a390-3b361834e615","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"model":"llm/e2e-unconfigured-facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T20:54:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"af9b8e6b-c711-4957-bae4-55cbe8c23515","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"d57e58f7-13c2-41b4-af25-41bccc0fc66d","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:54:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"03dc6662-a50f-45ee-87f6-aaaaa25f1da2","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"03dc6662-a50f-45ee-87f6-aaaaa25f1da2","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"286ac39f-9b6d-47c9-8302-dc12198a5ab5","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:54:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"286ac39f-9b6d-47c9-8302-dc12198a5ab5","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:54:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9d5787ba-f128-41e4-af5c-ff68185d71aa","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"eee193de-6caa-4af7-aa22-d96a76f1669c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:54:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e27ae912-26b0-4ff2-9078-55ae4882eda0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"99468d50-9f1f-4bfa-b184-4cfe3eb607b0","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"99468d50-9f1f-4bfa-b184-4cfe3eb607b0","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"058f4871-b378-4cef-8a5a-bd4c2929cc36","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"6c4f3bbc-42e0-446e-a987-905c8424279b","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:02Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:55:02Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"1d033cb8-caaa-4d6f-8b63-26b046533935","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:55:02Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"1d033cb8-caaa-4d6f-8b63-26b046533935"}
{"level":"info","ts":"2026-07-03T20:55:02Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"1d033cb8-caaa-4d6f-8b63-26b046533935","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:55:02Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"4b622725-3f45-47e6-ad66-8273645de091","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"ede1cae6-ae04-4bbe-a8e5-50e645fdd536","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"67144a52-cad2-4132-8c4d-bc96fdd0bb39","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"a8353ff4-e4d5-4a08-9d1e-346ac207b497","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"a8353ff4-e4d5-4a08-9d1e-346ac207b497","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"a9e83329-0539-4219-83d2-ca87b1b54cc3","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:07Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"a1024b28-9cfd-4d58-84cc-5ac5e9a62544","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-03T20:55:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"38d281b3-216f-4d52-a45e-6049c7122315","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"3eddde6c-255b-4262-9dd6-e7449e948c52","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"f8065f5f-4cba-487c-9a48-4ad7b074abf6","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"f8065f5f-4cba-487c-9a48-4ad7b074abf6","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"6f0fb590-0c88-440b-bd48-ed0924e31b00","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"6f0fb590-0c88-440b-bd48-ed0924e31b00","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ea0dc092-9f7b-4c3d-9e13-d45ef9d77a9d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"29ae7f10-b3d7-43e9-8482-b8ac0a8acb1a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:11Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:55:11Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"0f7fad85-b88c-49bc-a309-67df08257057","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:55:11Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"0f7fad85-b88c-49bc-a309-67df08257057"}
{"level":"info","ts":"2026-07-03T20:55:11Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"0f7fad85-b88c-49bc-a309-67df08257057","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:55:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7123c7d0-b564-424a-9cdc-5909d2b12a7c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"fd636ed6-3178-41d2-94f9-bb890c757a48","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"fd636ed6-3178-41d2-94f9-bb890c757a48","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:15Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"1eed8c65-c5c2-4ad2-8b70-eb9bb8b2aad2","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T20:55:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3edc7b16-e7a5-4d5a-85df-1f51da9c72b8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ff617971-ca5e-46e6-af57-c9eacf3bd0ab","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"a01a133a-83a3-4e5f-8435-557ae54226a2","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"a01a133a-83a3-4e5f-8435-557ae54226a2","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"877f696e-4cab-4ced-a7c9-d4759fe015b3","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"877f696e-4cab-4ced-a7c9-d4759fe015b3","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"5db4fd5c-4fbf-43b1-b646-2c8756b39f07","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"5db4fd5c-4fbf-43b1-b646-2c8756b39f07","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:55:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:55:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b076118e-f2ba-4047-a216-c8681a401d76","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"20f151be-7e2c-49e8-9794-49dbb1ebaa43","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"545a227d-e1af-4908-b0b7-30af81927aa3","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"545a227d-e1af-4908-b0b7-30af81927aa3","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"82a43f4e-f859-4db7-abfa-d2f1588f40c5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"80a82c5c-1551-4ba7-b866-8bf96d5751f0","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"80a82c5c-1551-4ba7-b866-8bf96d5751f0","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"1f39c324-d348-4690-b997-9cda44ae6e66","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"a2de88d2-b139-4a25-a045-8a29d70a9367","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"4e578f59-a465-4af6-afcf-26833b1a6f55","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"4e578f59-a465-4af6-afcf-26833b1a6f55","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:32Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3cdc89d8-b3db-4658-b57c-c04504a0995c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"d739ea6c-8313-42a8-802a-bb8627605665","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"d739ea6c-8313-42a8-802a-bb8627605665","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"451496af-4cc2-4226-af66-507bd451e9f1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:55:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"b65bf7ac-b635-4500-99cb-45128bf88564","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"b65bf7ac-b635-4500-99cb-45128bf88564","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:40Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"04a311e0-607d-48fc-b3d6-5a8310be383e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:55:40Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:55:40Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"30be2887-1382-4b2f-8131-221568326c00","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:55:40Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"30be2887-1382-4b2f-8131-221568326c00"}
{"level":"info","ts":"2026-07-03T20:55:40Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"30be2887-1382-4b2f-8131-221568326c00","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T20:55:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"f1172a13-587f-40b2-8a16-b5a9ce8683ca","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"f1172a13-587f-40b2-8a16-b5a9ce8683ca","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"1bdf9304-58ee-440c-9053-99d191cf1907","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"1bdf9304-58ee-440c-9053-99d191cf1907","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"e897351d-0dd8-456e-adb5-79ad906b2933","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"e897351d-0dd8-456e-adb5-79ad906b2933","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"d033e8e0-638d-4fc4-98aa-8390073a0484","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"d033e8e0-638d-4fc4-98aa-8390073a0484","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5a79a0c2-698d-4d36-92cd-8b704d879b2f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"5a79a0c2-698d-4d36-92cd-8b704d879b2f","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:55:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:55:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:55:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"238610d5-a07b-4370-aff9-248f9fa22a0a","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"238610d5-a07b-4370-aff9-248f9fa22a0a","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"45193ec6-1b43-4410-88f9-a1d4b3062ebd","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:55:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"45193ec6-1b43-4410-88f9-a1d4b3062ebd","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"4b64f4c9-f2fe-4fae-aed2-a3b4d16dff7d","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"4b64f4c9-f2fe-4fae-aed2-a3b4d16dff7d","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"d6081b6a-4936-4be4-a897-8cecfb98e1f5","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"d6081b6a-4936-4be4-a897-8cecfb98e1f5","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"58bce91a-210e-44cb-9939-f3c5baf04e35","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"58bce91a-210e-44cb-9939-f3c5baf04e35","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"d2edba6d-0ee5-4879-b82d-44bee8d79241","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-8c0aa8e8","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-8c0aa8e8","reconcileID":"d2edba6d-0ee5-4879-b82d-44bee8d79241","error":"failed to revoke API keys for subscription e2e-filter-sub-a-8c0aa8e8 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"b4c832c6-8b1d-4245-a0ac-24b976272c5e","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"b4c832c6-8b1d-4245-a0ac-24b976272c5e","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b164f220-ce50-4681-9048-71a11d94a139","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b164f220-ce50-4681-9048-71a11d94a139","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"155e155c-c7c5-467a-9472-11de7c645f33","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"155e155c-c7c5-467a-9472-11de7c645f33","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:56:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"88b5b2bd-dded-4680-9439-6f081d2fc686","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:56:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"cbc69e34-2dce-4e24-9270-d6db9fc2fd96","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:56:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"46007292-e48b-4c3c-bab0-dd2b060d0095","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"46007292-e48b-4c3c-bab0-dd2b060d0095","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:56:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:56:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:56:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"d9087b8e-3578-437a-aa92-eb580ba7013e","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"d9087b8e-3578-437a-aa92-eb580ba7013e","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"f83b5def-0c49-4b89-85b2-b629e6077b55","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"f83b5def-0c49-4b89-85b2-b629e6077b55","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"7d348644-de52-4266-a9ae-0bdd09f405ab","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"7d348644-de52-4266-a9ae-0bdd09f405ab","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"ff98340d-5089-4dad-8269-ff9def3d86c5","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"ff98340d-5089-4dad-8269-ff9def3d86c5","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"8c5b010f-e3d0-49c8-a7ab-10b6b76807c8","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"8c5b010f-e3d0-49c8-a7ab-10b6b76807c8","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:56:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"0680d42d-9b9f-40c1-bdc4-cfe5b8ee05f8","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:56:38Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"b47b1973-684a-42b4-b099-4edbe3450378","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-ordering-sub","models-as-a-service/premium-simulator-subscription"]}
{"level":"info","ts":"2026-07-03T20:56:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"441a3150-0013-4e86-81dc-197e35dc7f34","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:56:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"abd5bf9f-4525-4e8d-a481-20b397567481","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:56:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"70a23c0d-b75f-4b7f-9a4d-d72104483e16","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:56:39Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:56:39Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"6e5cad74-8024-4aeb-8b49-ffb82027556b","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:56:39Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"6e5cad74-8024-4aeb-8b49-ffb82027556b"}
{"level":"info","ts":"2026-07-03T20:56:39Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"6e5cad74-8024-4aeb-8b49-ffb82027556b","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:56:39Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"36818e61-9055-4e34-908c-c01efdb3d494","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:56:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"425156f3-fa32-4894-abd6-83bc90af920a","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"425156f3-fa32-4894-abd6-83bc90af920a","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:44Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c019afd5-ea92-4eab-945a-0da7b8b41f7f","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:44Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c019afd5-ea92-4eab-945a-0da7b8b41f7f","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:56:44Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"43ddaf49-6112-4bb3-b2fc-ee059e33b049","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:56:47Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"8882c8e2-7bc5-4680-b200-830e49296b0f","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:47Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"8882c8e2-7bc5-4680-b200-830e49296b0f","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:50Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"87e774d8-8465-4c34-9130-e3ac03419cfa","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"87e774d8-8465-4c34-9130-e3ac03419cfa","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:53Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"ead8a91d-db1b-4f4b-925b-b533c007438b","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"ead8a91d-db1b-4f4b-925b-b533c007438b","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:56:53Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"949e46ce-dbb9-4c0b-a5b5-7c59ceb848ce","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1,"subscriptions":["models-as-a-service/premium-simulator-subscription"]}
{"level":"info","ts":"2026-07-03T20:56:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:56:53Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:56:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"747d38ad-d044-4f17-a39a-c134d9c2f3ff","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"747d38ad-d044-4f17-a39a-c134d9c2f3ff","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"e9f6cfab-447c-495e-b4b9-1f8e1bc9e765","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:56:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"e9f6cfab-447c-495e-b4b9-1f8e1bc9e765","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"9ee6f4e1-5ff8-49bb-8dbf-f1192b115f99","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"9ee6f4e1-5ff8-49bb-8dbf-f1192b115f99","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"89d50e39-03fa-4e94-a60e-2e1c504b1f87","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"89d50e39-03fa-4e94-a60e-2e1c504b1f87","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:57:05Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"b5d2baca-d59f-474a-8df5-9308fb8985a2","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T20:57:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"160646c5-f546-4cba-b301-14e3a5cdb027","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"160646c5-f546-4cba-b301-14e3a5cdb027","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"7f2a0f01-c1dc-44bb-8646-33a9b0f797db","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-15eb1c6b","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-15eb1c6b","reconcileID":"7f2a0f01-c1dc-44bb-8646-33a9b0f797db","error":"failed to revoke API keys for subscription e2e-filter-sub-b-15eb1c6b after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:57:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"f12141ab-121b-4b88-b748-d54e294ec168","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:57:14Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"ea3171ed-1414-45d3-b7f1-f889d2ae29f2","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-03T20:57:14Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"7c053c91-af58-4465-aeab-0c7ef7bab40e","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:57:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"fabda0e1-f019-44da-9b66-2aeed20bcf81","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"fabda0e1-f019-44da-9b66-2aeed20bcf81","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"71dd9f2b-41a2-4d84-802e-3fdd20e852bf","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"71dd9f2b-41a2-4d84-802e-3fdd20e852bf","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"6341a786-4f3f-489d-aae2-6b298bd805f6","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-eb9fd595","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-eb9fd595","reconcileID":"6341a786-4f3f-489d-aae2-6b298bd805f6","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-eb9fd595 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"22cf25e7-d698-4436-8485-639ef9451005","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"22cf25e7-d698-4436-8485-639ef9451005","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:57:23Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"dcd93cbb-a7a8-4b78-b931-92ad5168d8e1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T20:57:23Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T20:57:23Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"82dd2604-1e9c-4f1e-9172-30cd0ca2f6c5","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T20:57:23Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"82dd2604-1e9c-4f1e-9172-30cd0ca2f6c5"}
{"level":"info","ts":"2026-07-03T20:57:23Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"82dd2604-1e9c-4f1e-9172-30cd0ca2f6c5","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T20:57:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T20:57:23Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T20:57:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"72dbbbfd-2b8b-42bc-999e-ee4f684fb7f4","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"72dbbbfd-2b8b-42bc-999e-ee4f684fb7f4","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T20:57:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3c513cd0-007e-4ae5-a04f-dd48984099cf","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T20:57:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"089c3eee-b07e-45cb-b816-0918402ec02f","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-f30784b7","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-f30784b7","reconcileID":"089c3eee-b07e-45cb-b816-0918402ec02f","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-f30784b7 after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a58f380b-566a-4ece-8f2f-c207db34210c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a58f380b-566a-4ece-8f2f-c207db34210c","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"63a0c99b-8770-4a5e-b4ae-ef3792c6d701","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:860\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:360\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T20:57:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"63a0c99b-8770-4a5e-b4ae-ef3792c6d701","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.opendatahub.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": tls: failed to verify certificate: x509: certificate signed by unknown authority","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}