{"level":"info","ts":"2026-07-03T19:28:48Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"eadd0ad0-e34c-4546-abdc-d8b6f4ec9833","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"437b83fc-ce00-4b66-b621-db07ba3aad2e","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"437b83fc-ce00-4b66-b621-db07ba3aad2e"}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"437b83fc-ce00-4b66-b621-db07ba3aad2e","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"cbaf3242-1d9b-4baa-a287-4b191f71b901","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:28:48Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"d1ca6b56-dc2f-4dc8-a836-b691b177aa48","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"12cf5e23-5fab-4ee4-bf07-b312abc9c57d","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bfe3d8d5-5765-4da3-876e-42adf1dc625c","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"d1ca6b56-dc2f-4dc8-a836-b691b177aa48"}
{"level":"info","ts":"2026-07-03T19:28:48Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"d1ca6b56-dc2f-4dc8-a836-b691b177aa48","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:28:48Z","msg":"failed to update MaaSAuthPolicy status","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"d1ca6b56-dc2f-4dc8-a836-b691b177aa48","name":"e2e-neg-ghost-sub-auth-c537128a","error":"Operation cannot be fulfilled on maasauthpolicies.maas.opendatahub.io \"e2e-neg-ghost-sub-auth-c537128a\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSAuthPolicyReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasauthpolicy_controller.go:1702\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSAuthPolicyReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasauthpolicy_controller.go:534\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:48Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"1d243156-3a5b-4820-9a96-6d712a0a8206","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:48Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"1d243156-3a5b-4820-9a96-6d712a0a8206","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:28:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:28:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:28:51Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"b037a55f-8138-4ac6-8dbb-6dc49c1a0118","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:51Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"b037a55f-8138-4ac6-8dbb-6dc49c1a0118","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:54Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"2b788d09-b74b-45b0-815c-aaf225c61d79","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:54Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"2b788d09-b74b-45b0-815c-aaf225c61d79","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:57Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"f05d72a0-4770-4070-950c-2561b776f9a4","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:28:57Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"f05d72a0-4770-4070-950c-2561b776f9a4","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"f5c5fbc6-e54c-4fc9-9010-496b5c3588c7","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"f5c5fbc6-e54c-4fc9-9010-496b5c3588c7","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"46a8d490-b2d6-4cb8-afbc-9ce6ced4d7e4","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"46a8d490-b2d6-4cb8-afbc-9ce6ced4d7e4","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c5394a6b-b20f-48e0-8122-36b0519a8742","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:04Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"23ef4989-608d-4e43-8acc-d69ff45d8efc","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-neg-ghost-sub-c537128a","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-03T19:29:04Z","msg":"model/route not found during cleanup, deleting TokenRateLimitPolicy via labels","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"23ef4989-608d-4e43-8acc-d69ff45d8efc","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-c537128a","error":"MaaSModelRef not found: llm/nonexistent-model-c537128a"}
{"level":"info","ts":"2026-07-03T19:29:04Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e9e795d8-58c6-44a3-9006-b5f6962887c5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:05Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"34c8c85a-1e7c-4d9b-ae4d-b65c91dbef84","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:29:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"2cc84ef2-a184-4352-934d-aadcd76014df","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"2cc84ef2-a184-4352-934d-aadcd76014df","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"2b2dcf3e-1ab7-4d41-9309-90a3c1576bdc","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"2b2dcf3e-1ab7-4d41-9309-90a3c1576bdc","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"cc53e92e-b55e-4f96-895a-e548c505146d","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"cc53e92e-b55e-4f96-895a-e548c505146d","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"11bd9901-2759-4474-b79e-17af4ccac15a","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"11bd9901-2759-4474-b79e-17af4ccac15a","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"3c66099b-471a-4a93-960e-bc97da5aa4d3","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"3c66099b-471a-4a93-960e-bc97da5aa4d3","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:19Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"708315c4-e824-4e7c-991e-05284ff2d0b2","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-03T19:29:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:29:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:29:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"4adce8e0-8796-4a91-bc19-aa420dd12cea","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"4adce8e0-8796-4a91-bc19-aa420dd12cea","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"cdbd5a49-9f2d-481c-96b2-baa86bf8c0f4","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"cdbd5a49-9f2d-481c-96b2-baa86bf8c0f4","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"9a17f616-fe6f-4b12-8dc0-541cd494df8b","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"9a17f616-fe6f-4b12-8dc0-541cd494df8b","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:28Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"d92ee80f-10a8-4042-abe6-f0fe6539c323","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:29:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"b7c2383c-610d-4d3d-8e6f-189f35f711bb","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"b7c2383c-610d-4d3d-8e6f-189f35f711bb","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"9006a9ad-4e7f-454d-b3eb-6ccc5708db7b","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"9006a9ad-4e7f-454d-b3eb-6ccc5708db7b","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:35Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-auth-c537128a","reconcileID":"850ca9a9-4d48-4786-b8c8-4974dc90882f","MaaSAuthPolicy":{"name":"e2e-neg-ghost-sub-auth-c537128a","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T19:29:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"57f491b3-8b23-47ec-808d-c7001a1380b9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:35Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c146d861-f449-40f3-9c6d-44f7b5abd4a1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b2f344be-a5d0-4801-83fd-d9d870212eba","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:36Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"451f30ee-7a00-457a-8c1b-381a24a4c2c1","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"eeb6ead5-eca8-45a0-8967-3a9ea2a25723","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"eeb6ead5-eca8-45a0-8967-3a9ea2a25723"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"eeb6ead5-eca8-45a0-8967-3a9ea2a25723","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"model not found, cleaning up generated AuthPolicy","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"eeb6ead5-eca8-45a0-8967-3a9ea2a25723","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-4c7a401c"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b6f24047-6af6-4e14-901f-207b35e9fc96","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:36Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"5fef31bd-9711-439f-b9af-887a798586bf","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"5fef31bd-9711-439f-b9af-887a798586bf"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"5fef31bd-9711-439f-b9af-887a798586bf","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"model not found, cleaning up generated AuthPolicy","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"5fef31bd-9711-439f-b9af-887a798586bf","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-4c7a401c"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"a4213e76-02b1-44ca-abbc-554f5af95ff4","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ab4c15bf-7245-4cf4-aa4d-9c90947d2147","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-policy-4c7a401c","reconcileID":"a4213e76-02b1-44ca-abbc-554f5af95ff4","MaaSAuthPolicy":{"name":"e2e-neg-ghost-policy-4c7a401c","namespace":"models-as-a-service"},"model":"llm/nonexistent-model-4c7a401c"}
{"level":"info","ts":"2026-07-03T19:29:36Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0996e9fc-09dd-4e4c-90df-7dfee7c29918","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:29:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"36e5b36a-08a0-45b6-9591-dd239295ca2d","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"36e5b36a-08a0-45b6-9591-dd239295ca2d","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"cc4c99aa-3a5e-44e7-a95f-001e59eb7261","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:29:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"ad64ed47-351d-4d49-9db2-6bed823a3345","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"ad64ed47-351d-4d49-9db2-6bed823a3345","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"8124ae8d-da70-4b9a-a74f-ff8ea4344778","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"8124ae8d-da70-4b9a-a74f-ff8ea4344778","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"f0dc8434-5547-4f1b-a42d-b8e17c9eaafd","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"f0dc8434-5547-4f1b-a42d-b8e17c9eaafd","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"7d417e44-ea63-4f1c-b6bf-4c402293394e","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:29:49Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"0dde727a-7c61-4908-b357-6cc4a5771fca","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:29:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"60655a5e-3ded-4da1-882d-be542d5ef2fd","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"60655a5e-3ded-4da1-882d-be542d5ef2fd","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:29:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:29:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:29:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"20833a63-91e3-41bb-950d-79156ad6d456","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"20833a63-91e3-41bb-950d-79156ad6d456","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"85eb3af4-b8b5-4396-850f-755e9417a9a2","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"85eb3af4-b8b5-4396-850f-755e9417a9a2","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"11b63a80-febf-480a-a8c4-69863fcbe66a","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:29:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"11b63a80-febf-480a-a8c4-69863fcbe66a","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"f18b3a55-d5be-4c9b-b3e3-db73d9d4a038","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"f18b3a55-d5be-4c9b-b3e3-db73d9d4a038","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"fad12ebf-423a-41e7-a2ba-60b2173936cd","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"fad12ebf-423a-41e7-a2ba-60b2173936cd","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"7738b2aa-bca0-43ad-adc7-148c1fe29ae5","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"7738b2aa-bca0-43ad-adc7-148c1fe29ae5","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3e572af1-57eb-4f73-9c43-319dc94908f4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:07Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"e52a264d-4f34-4d87-ae31-a9f79b46b888","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-apikey-sub-binding-4a58c111","models-as-a-service/simulator-subscription"]}
{"level":"info","ts":"2026-07-03T19:30:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d91f46f6-ab9a-4f44-8092-c8d262b12f14","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"96f59371-a160-45a3-bdd1-766b968b0473","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:30:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"92c1189b-bc00-4792-ac62-cecf0661d381","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"92c1189b-bc00-4792-ac62-cecf0661d381","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"c2e23441-ad8d-4131-959b-6b8ea756a48f","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"c2e23441-ad8d-4131-959b-6b8ea756a48f","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"e8d8ce72-31fb-4031-bafd-362db3b7f6dc","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"e8d8ce72-31fb-4031-bafd-362db3b7f6dc","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"18b1b14e-a3f5-4cd0-b87b-956b6d7b83c8","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"18b1b14e-a3f5-4cd0-b87b-956b6d7b83c8","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:30:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:30:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"f77b0f45-6571-43b8-9318-a2acf27631c4","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"f77b0f45-6571-43b8-9318-a2acf27631c4","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"1f35aad5-cdb8-4787-85fb-651207fa6518","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"1f35aad5-cdb8-4787-85fb-651207fa6518","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:25Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"47b7d4fc-c2cb-41de-8684-c1bcff14d689","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/simulator-subscription"]}
{"level":"error","ts":"2026-07-03T19:30:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"48663cea-3a97-4eb3-b1ba-5a7d4353a87b","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"48663cea-3a97-4eb3-b1ba-5a7d4353a87b","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:28Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"36286418-6ab9-46f6-bbc4-f931121170f1","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:30:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"d7665f7a-1df3-45fa-8b2c-d6755e29659a","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"d7665f7a-1df3-45fa-8b2c-d6755e29659a","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"c7b78e74-5c63-42f3-a212-87149c8c4853","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"c7b78e74-5c63-42f3-a212-87149c8c4853","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"a48131b4-9eb9-40a3-abc2-df70aa220441","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"a48131b4-9eb9-40a3-abc2-df70aa220441","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"aea1d8b8-852a-4c8a-9eb0-d2ff666296fa","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:38Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:30:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"9100eb6e-90aa-4052-a8ea-bd493a833772","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:38Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"43ecfb0d-9ee0-4859-89cd-9247f6770fa7","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:30:38Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"43ecfb0d-9ee0-4859-89cd-9247f6770fa7"}
{"level":"info","ts":"2026-07-03T19:30:38Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"43ecfb0d-9ee0-4859-89cd-9247f6770fa7","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:30:38Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"626fca31-9e2e-4b6a-9b91-cca8cea75fe4","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:30:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"17ee844f-5419-4e93-9099-36727accd26c","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"17ee844f-5419-4e93-9099-36727accd26c","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:42Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-auth-pass-sub-fail","reconcileID":"8c4472f9-2699-4a26-95e2-ad4e2d626442","MaaSAuthPolicy":{"name":"e2e-auth-pass-sub-fail","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-03T19:30:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"7ea6743f-d210-4baa-be7d-ba10d63b50d6","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:42Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"4995ddfd-56d0-4607-8663-2ac88f5a5051","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:30:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"d0e6ec4c-b20e-4587-a916-ddb73959a4b0","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"d0e6ec4c-b20e-4587-a916-ddb73959a4b0","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"b16deb4c-0574-49eb-ad6c-a11691ffeb90","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"b16deb4c-0574-49eb-ad6c-a11691ffeb90","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:46Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"b147497d-6aff-4185-8f6e-40bd20474243","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"d99a2b4b-1a75-4c73-b984-88924e059f4b","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"156befd0-fb3b-4652-8745-0f8248fb4bc4","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:47Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"cb82f985-96aa-4071-aa79-06af64c5f7b0","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"cb82f985-96aa-4071-aa79-06af64c5f7b0"}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"cb82f985-96aa-4071-aa79-06af64c5f7b0","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"7d703ae9-a2cf-4af0-9ccd-58f674980dca","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:47Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"3172bb51-e2c2-49db-bece-2678112763e7","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"3172bb51-e2c2-49db-bece-2678112763e7"}
{"level":"info","ts":"2026-07-03T19:30:47Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"3172bb51-e2c2-49db-bece-2678112763e7","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:30:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"57508d77-bc7e-4406-aeeb-97e51ea05f32","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"57508d77-bc7e-4406-aeeb-97e51ea05f32","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:30:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:30:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"cf490bc3-9064-4514-a85f-92a189687ad1","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"3b869637-138a-4446-8ed1-a3f204399422","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:30:51Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:30:51Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"fa20d4f2-2d53-476a-bbf0-a55130f2a9d9","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"error","ts":"2026-07-03T19:30:51Z","msg":"failed to update MaaSModelRef status","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"3b869637-138a-4446-8ed1-a3f204399422","name":"e2e-unconfigured-facebook-opt-125m-simulated","error":"Operation cannot be fulfilled on maasmodelrefs.maas.opendatahub.io \"e2e-unconfigured-facebook-opt-125m-simulated\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatusWithReason\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:381\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).updateStatus\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:345\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSModelRefReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maasmodelref_controller.go:185\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:51Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"fa20d4f2-2d53-476a-bbf0-a55130f2a9d9"}
{"level":"info","ts":"2026-07-03T19:30:51Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"fa20d4f2-2d53-476a-bbf0-a55130f2a9d9","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:30:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"d1b977a1-7cec-4d74-9b11-c3bd6a9ae3a9","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"d1b977a1-7cec-4d74-9b11-c3bd6a9ae3a9","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:30:52Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"a579f67f-f901-47b8-9352-3e8a8af9340c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:30:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"a4e7cbce-08b4-4a61-8924-a6857c1171a6","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"a4e7cbce-08b4-4a61-8924-a6857c1171a6","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"24e9767a-66ed-4f66-a24c-6860ee5aac8a","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:30:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"24e9767a-66ed-4f66-a24c-6860ee5aac8a","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"62aaf6ba-a17d-48a6-92b4-32f6a6854c18","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"62aaf6ba-a17d-48a6-92b4-32f6a6854c18","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"c29e4e0c-fca3-487c-8159-12d91b2c74c0","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"c29e4e0c-fca3-487c-8159-12d91b2c74c0","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"338d38dc-90e2-44ee-9e56-8f53a14d72d0","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"338d38dc-90e2-44ee-9e56-8f53a14d72d0","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"509a8c26-248c-4e7f-b968-71e4f682e3b1","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"509a8c26-248c-4e7f-b968-71e4f682e3b1","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:31:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"72328584-f857-4e92-954c-9cb084b2718b","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:10Z","msg":"TokenRateLimitPolicy created","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"ae0d9e48-d848-493e-9b39-8a96edc851bc","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated","model":"e2e-unconfigured-facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/e2e-rate-limit-test-subscription"]}
{"level":"info","ts":"2026-07-03T19:31:10Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"81bd7140-e62b-4a0a-ac3e-e6532469f642","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"bded0e21-0b1c-4ef7-af1a-cfbdae0de8ff","MaaSModelRef":{"name":"e2e-distinct-simulated","namespace":"llm"},"routeName":"e2e-distinct-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-distinct-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-trlp-test-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-trlp-test-simulated","reconcileID":"106f3813-6d35-45a6-b233-4b2d0903dd60","MaaSModelRef":{"name":"e2e-trlp-test-simulated","namespace":"llm"},"routeName":"e2e-trlp-test-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-trlp-test-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"49803885-3d2c-439c-80b5-4b4664b73191","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9b195d43-54c7-4042-a532-888853dbc4a5","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"3ef84f2b-2a7b-4e00-805e-d3a337b08c7b","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"40e6f657-b1e2-46e3-88c9-7399499d98aa","MaaSModelRef":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"routeName":"e2e-distinct-2-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-distinct-2-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"bcb206fc-7dd8-4f2f-8510-609e77df550b","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"bcb206fc-7dd8-4f2f-8510-609e77df550b"}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"bcb206fc-7dd8-4f2f-8510-609e77df550b","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:31:13Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"25351418-a1e6-4d18-8d13-1c85dce96795","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"25351418-a1e6-4d18-8d13-1c85dce96795"}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-access","reconcileID":"25351418-a1e6-4d18-8d13-1c85dce96795","MaaSAuthPolicy":{"name":"premium-simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:31:13Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"ec277cb3-c33d-47fb-8d61-f16fa95fa43a","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"ec277cb3-c33d-47fb-8d61-f16fa95fa43a"}
{"level":"info","ts":"2026-07-03T19:31:13Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"ec277cb3-c33d-47fb-8d61-f16fa95fa43a","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:31:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"56adf71f-d2d5-41a9-ae23-5dab07127096","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"56adf71f-d2d5-41a9-ae23-5dab07127096","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"0f9b6c99-ee47-4fcf-8ef6-205b9d483f5d","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"0f9b6c99-ee47-4fcf-8ef6-205b9d483f5d","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:31:17Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"62b91e0c-2515-47d6-b340-556993fbad61","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:17Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:17Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"c3a4e5ef-8bdb-4a92-9db7-90e29986a7dd","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:17Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"c3a4e5ef-8bdb-4a92-9db7-90e29986a7dd"}
{"level":"info","ts":"2026-07-03T19:31:17Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"c3a4e5ef-8bdb-4a92-9db7-90e29986a7dd","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:31:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"01fac2e1-ed8b-48ae-b72e-5086b1aeb298","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"01fac2e1-ed8b-48ae-b72e-5086b1aeb298","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:31:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:31:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:31:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"1e22a8d1-b819-445a-af5d-6f6ce503b55b","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"1e22a8d1-b819-445a-af5d-6f6ce503b55b","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"376d65d4-03f5-4241-9f87-da789aa85285","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"376d65d4-03f5-4241-9f87-da789aa85285","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"d9836290-d5d9-4048-a7c3-ed306e322349","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"d9836290-d5d9-4048-a7c3-ed306e322349","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"928f3686-cb50-4036-8feb-acceafa955dd","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"928f3686-cb50-4036-8feb-acceafa955dd","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"ec171096-7692-4ec1-a5f4-308fabdaca61","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"ec171096-7692-4ec1-a5f4-308fabdaca61","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"266d511d-7204-4c0f-b959-5aff78cc6d4c","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"266d511d-7204-4c0f-b959-5aff78cc6d4c","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:31:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"5712f8a8-ede2-4a5b-b4a5-fb402a7b324f","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T19:31:37Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"ba76edf6-7776-4a5b-83bd-175bfe8d5f5c","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:31:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"2715a8f5-6b7e-448a-bd44-e31e8049ec58","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"2715a8f5-6b7e-448a-bd44-e31e8049ec58","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"7f09b2f4-1874-41aa-bd87-89d60fc058f5","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"7f09b2f4-1874-41aa-bd87-89d60fc058f5","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"0ec866f2-948c-4b77-b8a9-9e0779c589d6","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"0ec866f2-948c-4b77-b8a9-9e0779c589d6","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:31:47Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-auth","reconcileID":"79a8bd9a-1cf6-486b-8a92-08508bdf9fbc","MaaSAuthPolicy":{"name":"e2e-rate-limit-test-auth","namespace":"models-as-a-service"},"model":"llm/e2e-unconfigured-facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T19:31:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"8b07596d-40c1-4149-9f16-0cc0017a4852","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:47Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"9301fd8e-60ce-4592-93c9-e5c3583cec45","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:31:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"afea4a13-cddb-449c-b259-96aba3287c07","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"afea4a13-cddb-449c-b259-96aba3287c07","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:31:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:31:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"2e70bde8-cc4e-4b2b-a41f-cfb84965c68e","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"ee20adee-6dfa-4d4f-a72d-9f95721bf6f0","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:51Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a16ca134-87e2-4933-ace1-7083ce29ace2","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a16ca134-87e2-4933-ace1-7083ce29ace2"}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a16ca134-87e2-4933-ace1-7083ce29ace2","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"c1e01f6e-bc74-40d9-b72b-b6d706294c32","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:51Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"7121c33b-c15e-426e-9598-6261f36668fc","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"7121c33b-c15e-426e-9598-6261f36668fc"}
{"level":"info","ts":"2026-07-03T19:31:51Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"7121c33b-c15e-426e-9598-6261f36668fc","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:31:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"69348055-0c10-4cdd-b4ec-8281ef0662e1","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:52Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"a91047bc-58fa-413e-b0af-4ccac2c3154a","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:31:52Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:31:52Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"878a2246-4e73-4ab7-bce2-a7ef413040a7","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:31:52Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"878a2246-4e73-4ab7-bce2-a7ef413040a7"}
{"level":"info","ts":"2026-07-03T19:31:52Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"878a2246-4e73-4ab7-bce2-a7ef413040a7","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:31:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"fa4d9a4f-ebef-4a78-99fe-d4c9d97702c1","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"fa4d9a4f-ebef-4a78-99fe-d4c9d97702c1","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"65f25354-8d49-4f2e-92a1-2cdaa5dcd0f7","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"65f25354-8d49-4f2e-92a1-2cdaa5dcd0f7","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"00f1bc6f-f6da-49a8-be17-8c8bb07039f8","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:31:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"00f1bc6f-f6da-49a8-be17-8c8bb07039f8","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"954f4adc-90d2-474e-a6c7-9828eee8f92d","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"954f4adc-90d2-474e-a6c7-9828eee8f92d","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"91597c56-4249-4e53-a0e8-2ccd523fada0","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"91597c56-4249-4e53-a0e8-2ccd523fada0","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"5e59476c-7854-43da-a6dd-188873a90b7b","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"5e59476c-7854-43da-a6dd-188873a90b7b","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"ecfae3e0-b496-4bba-9376-fe44a3195b25","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"ecfae3e0-b496-4bba-9376-fe44a3195b25","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"62c52506-6171-4b40-a3bc-6d0a9fed9762","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"62c52506-6171-4b40-a3bc-6d0a9fed9762","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"a8a32926-9a41-47d1-887b-469364d68fa1","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:13Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"4596a3dd-4263-4302-8b0e-cf24851a4f81","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated","model":"llm/e2e-unconfigured-facebook-opt-125m-simulated","subscriptionCount":1,"subscriptions":["models-as-a-service/e2e-models-exempt-test-subscription"]}
{"level":"info","ts":"2026-07-03T19:32:13Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"97f2aa01-c6dd-4891-b500-debdea2db8b3","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"8f8f1c9f-e695-409b-8dc2-ff4a40c0c6f8","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:15Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:32:15Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a4ff7eb5-0016-41e9-b4ec-b0a789cfd4e0","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:32:15Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a4ff7eb5-0016-41e9-b4ec-b0a789cfd4e0"}
{"level":"info","ts":"2026-07-03T19:32:15Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"a4ff7eb5-0016-41e9-b4ec-b0a789cfd4e0","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:32:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"bdb7e74e-6e54-4962-8dca-66f9db0ffb8d","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"bdb7e74e-6e54-4962-8dca-66f9db0ffb8d","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:32:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:32:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"85cebe45-b2f9-4e45-9485-c69dea0170e2","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"85cebe45-b2f9-4e45-9485-c69dea0170e2","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:32:22Z","msg":"Built platform params","controller":"tenant","controllerGroup":"maas.opendatahub.io","controllerKind":"Tenant","Tenant":{"name":"default-tenant","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"default-tenant","reconcileID":"3b398b98-aadf-4497-9cf5-b468265ed140","tenant":"models-as-a-service/default-tenant","tenantID":"","subscriptionNamespace":"models-as-a-service","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:32:22Z","logger":"MaaSAPIDeploymentName","msg":"Generated deployment name","tenantID":"","deploymentName":"maas-api"}
{"level":"error","ts":"2026-07-03T19:32:22Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"1197a9ee-f784-4644-b89a-9bc1ff53bac6","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:22Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"1197a9ee-f784-4644-b89a-9bc1ff53bac6","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:25Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"b1f8d41d-d15c-4b3b-a426-8b8d88db3f05","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:25Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"b1f8d41d-d15c-4b3b-a426-8b8d88db3f05","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:28Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"4b249af8-4fda-4a93-85c0-35250244ccae","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:28Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"4b249af8-4fda-4a93-85c0-35250244ccae","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:31Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"ea30beef-ac3e-4f36-a681-4bd3d7c24923","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:31Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"ea30beef-ac3e-4f36-a681-4bd3d7c24923","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"039a6830-bb02-4544-b7d9-639096a772c1","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-facebook-opt-125m-simulated","model":"llm/facebook-opt-125m-simulated","subscriptionCount":1}
{"level":"info","ts":"2026-07-03T19:32:31Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"2ed10edb-92b8-4743-a98e-43bb358f419c","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:32:34Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"a0e1fc20-4f18-4fc3-b183-050463a4ef5f","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:34Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"a0e1fc20-4f18-4fc3-b183-050463a4ef5f","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:37Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"cda11fba-81ff-43c0-b551-66d9321bbb80","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:37Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"cda11fba-81ff-43c0-b551-66d9321bbb80","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:40Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"a6dec2f8-2c28-4848-8335-4f8d3d7e85a2","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:40Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"a6dec2f8-2c28-4848-8335-4f8d3d7e85a2","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:43Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"7936fcb3-1b69-47f6-8313-f7a5702136ea","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:43Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"7936fcb3-1b69-47f6-8313-f7a5702136ea","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:45Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-auth","reconcileID":"d639821c-81b6-4a50-9d71-1284818c1b48","MaaSAuthPolicy":{"name":"e2e-models-exempt-test-auth","namespace":"models-as-a-service"},"model":"llm/e2e-unconfigured-facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T19:32:45Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"3f98960e-18f4-4dfd-a733-1d003454a29b","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:46Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-unconfigured-facebook-opt-125m-simulated","reconcileID":"875feaae-6b45-4acf-869a-cdc5019f4f5e","MaaSModelRef":{"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"},"routeName":"e2e-unconfigured-facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"e2e-unconfigured-facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:32:46Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"1de9f29b-0204-4acf-954b-871f670ce6e0","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:46Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"1de9f29b-0204-4acf-954b-871f670ce6e0","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:32:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:32:49Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"3c640711-29e5-47b0-9867-23b6128829a9","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:49Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"3c640711-29e5-47b0-9867-23b6128829a9","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"e74800a9-b732-4313-ba90-6b208ec23d20","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"27559a88-6fa3-40f4-b3db-f0aa953325d8","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:50Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"c5c557ca-1374-4752-a458-525c1f60c349","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:32:52Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"a9181cd7-049e-4070-9a1c-00b0f889d6f1","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:52Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"a9181cd7-049e-4070-9a1c-00b0f889d6f1","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"76a1988d-7774-4b5f-bc30-cd6b68896f74","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"7ffadedc-7d2a-4e3d-a630-c41a4d14902d","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:54Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"09db29f8-4041-4df8-b860-692266271aac","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"09db29f8-4041-4df8-b860-692266271aac"}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"09db29f8-4041-4df8-b860-692266271aac","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"03cd2fc9-ef71-4734-9d48-9b8a30697f51","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"13e66ac4-fe7e-4673-8442-b46453d66a0f","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:54Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"f7dbf3f9-4f3b-4016-8a78-816a3eeef1fb","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:32:55Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"81f9c589-053d-42d4-919d-44d31eccadfa","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:55Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"81f9c589-053d-42d4-919d-44d31eccadfa","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"eb88b67d-07de-412f-b19a-2c20822de4c1","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:32:58Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"a9951007-33c6-41b6-88d9-381516e067d2","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:32:58Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"a9951007-33c6-41b6-88d9-381516e067d2","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:32:58Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-premium-sa-auth","reconcileID":"77738f30-5632-4dfb-8090-8d0d76ed7c0e","MaaSAuthPolicy":{"name":"e2e-premium-sa-auth","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-03T19:32:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"34f107dc-5c11-457b-9eeb-3bd4409e2c28","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:32:59Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"5de0c1b6-dd35-44d8-ac22-15340892a177","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:01Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"b9ffe062-4b38-4395-91d4-628a79569f1f","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:01Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"b9ffe062-4b38-4395-91d4-628a79569f1f","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d3e61cb0-7f03-4fb1-8b07-baae669c74c9","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:33:03Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:33:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"364304a2-f006-49b2-9d62-10cc0a5dd912","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:33:03Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"c5ea6178-95b7-48b4-98bb-505240781d2e","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:33:03Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"c5ea6178-95b7-48b4-98bb-505240781d2e"}
{"level":"info","ts":"2026-07-03T19:33:03Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"c5ea6178-95b7-48b4-98bb-505240781d2e","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:33:03Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b5a899fa-42af-4150-a117-b22b76ce7008","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:04Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"1b012e1b-fb12-45fa-8404-f95f86df3605","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:04Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"1b012e1b-fb12-45fa-8404-f95f86df3605","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bec0e23c-a8d3-45fb-afbe-2ff74efa3a19","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:33:07Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-extra-auth","reconcileID":"f7953f36-4713-47e1-9bbf-44c8f5487479","MaaSAuthPolicy":{"name":"e2e-extra-auth","namespace":"models-as-a-service"},"model":"llm/facebook-opt-125m-simulated"}
{"level":"info","ts":"2026-07-03T19:33:07Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"f8f438c9-a5de-4092-b064-108bb8c6a894","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:07Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"c2013a30-325f-41bd-a36e-6a04b035df71","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:07Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"c2013a30-325f-41bd-a36e-6a04b035df71","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:10Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"14b67304-1bde-412b-a6ac-d2006984f0d6","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:10Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"14b67304-1bde-412b-a6ac-d2006984f0d6","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:13Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"131df4c1-629e-448c-b7ab-ae61bcd99fe1","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:13Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"131df4c1-629e-448c-b7ab-ae61bcd99fe1","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9e4765c9-fd66-40b4-9e18-73d5c5d1157a","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:33:15Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"3258ac3e-4057-448c-9a19-f990846cf0f0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:16Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"53fb7363-da64-4769-94af-4c853f841671","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:16Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"53fb7363-da64-4769-94af-4c853f841671","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:33:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:33:19Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"d8691609-a005-4975-9f71-2e3162705ea4","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:19Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"8075c7fb-f39b-4d60-ba44-39903a0cb7d1","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:19Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"8075c7fb-f39b-4d60-ba44-39903a0cb7d1","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"adadda5b-bfba-4ece-a275-3aeef9d8bd26","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:33:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"ff845261-b8f8-414c-8f22-b74d1a9b2dd0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"c8ac5eee-ac43-4a26-8d02-eed635cc7b6c","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"c8ac5eee-ac43-4a26-8d02-eed635cc7b6c","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:24Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"bc0f3296-e765-4040-b0ea-557a0706a0f0","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"cea99813-7f55-41c8-b94d-aefd2fd80f11","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"cea99813-7f55-41c8-b94d-aefd2fd80f11","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:27Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"25bf397f-e1ad-4e0f-b0ad-112343a51984","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:33:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"ce3c157a-2721-41b6-9576-f7c516fa7827","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"ce3c157a-2721-41b6-9576-f7c516fa7827","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:31Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"9af4ff21-0661-42c8-90bd-8c56d01abc6b","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:33:31Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:33:31Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"9404ebf7-c76b-4c8f-ac5f-0d98f5acd920","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:33:31Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"9404ebf7-c76b-4c8f-ac5f-0d98f5acd920"}
{"level":"info","ts":"2026-07-03T19:33:31Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"9404ebf7-c76b-4c8f-ac5f-0d98f5acd920","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:33:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"d5da56a5-5daf-4f63-b81b-770009df10f8","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"d5da56a5-5daf-4f63-b81b-770009df10f8","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b1491e1a-a923-4f18-86c6-66cf07374d5e","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"b1491e1a-a923-4f18-86c6-66cf07374d5e","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:38Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"51de498d-e960-4c2e-a1a6-302a19897da3","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"51de498d-e960-4c2e-a1a6-302a19897da3","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"14ba6a2c-0c31-4c06-92ff-1d0a6aaa419f","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"14ba6a2c-0c31-4c06-92ff-1d0a6aaa419f","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:44Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c2c50176-eb31-4e23-aacc-55456c6b8a52","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:44Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c2c50176-eb31-4e23-aacc-55456c6b8a52","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:47Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"4156b6da-04c4-4363-b8c9-663a5a9faf10","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:47Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"4156b6da-04c4-4363-b8c9-663a5a9faf10","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:33:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:33:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:33:50Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"4f04f162-7f10-4401-8580-66ca68130e80","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"4f04f162-7f10-4401-8580-66ca68130e80","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:53Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"756dfe52-a7d0-4b1e-9444-a44f582b5200","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"756dfe52-a7d0-4b1e-9444-a44f582b5200","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"cabfb44d-c704-480f-8c0b-261be1caa188","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"cabfb44d-c704-480f-8c0b-261be1caa188","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"7d7648c7-b575-4942-b3c0-ab00e2a7592b","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:33:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"7d7648c7-b575-4942-b3c0-ab00e2a7592b","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"8b3097d9-4ed5-4dbc-8ec4-72d3f9430e5e","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-a-c8424e36","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-a-c8424e36","reconcileID":"8b3097d9-4ed5-4dbc-8ec4-72d3f9430e5e","error":"failed to revoke API keys for subscription e2e-filter-sub-a-c8424e36 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"bbc05a40-a4a3-4ab1-82a0-33f7cec1b11c","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"bbc05a40-a4a3-4ab1-82a0-33f7cec1b11c","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"46b157ba-01cd-4524-b7f1-20bdd4ee7771","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"46b157ba-01cd-4524-b7f1-20bdd4ee7771","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"60e5bce1-789f-4d78-81ea-845307dd5759","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"60e5bce1-789f-4d78-81ea-845307dd5759","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"a5a029ec-bd90-48c7-9350-92fbdd68bb66","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:34:11Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"d66e818c-8f70-4445-87ea-186b54eb34b6","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:34:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c5f9c8ec-e6e9-440f-ae0d-a999f88b3b08","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"c5f9c8ec-e6e9-440f-ae0d-a999f88b3b08","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"4b44d137-5857-473c-895c-63f26e18658d","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"4b44d137-5857-473c-895c-63f26e18658d","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:34:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:34:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"5dd89f2d-dccf-4843-8107-9981ee974283","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"5dd89f2d-dccf-4843-8107-9981ee974283","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"affd7af8-d23c-4bfa-a4aa-008e46977dd2","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:34:20Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"745ab37f-77ba-4dfb-a48f-b0c35ba9a86f","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":2,"subscriptions":["models-as-a-service/e2e-ordering-sub","models-as-a-service/premium-simulator-subscription"]}
{"level":"info","ts":"2026-07-03T19:34:20Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"379b9f44-78f6-47b1-a200-7ad14e8b2134","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:34:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"32c24198-99bf-4e4c-b8cd-ec473cdc823a","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:34:22Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:34:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"da5aaf0f-0349-4f72-9518-1847bd68976e","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:34:22Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"d222f723-f093-41f0-b3ff-068b279a43e1","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:34:22Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"d222f723-f093-41f0-b3ff-068b279a43e1"}
{"level":"info","ts":"2026-07-03T19:34:22Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"d222f723-f093-41f0-b3ff-068b279a43e1","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-07-03T19:34:22Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"231cf62d-08e7-4f89-8466-1ce7247cbc64","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:34:23Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"2929d0b1-3d9d-4051-9c11-38edd4dd406f","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:23Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"2929d0b1-3d9d-4051-9c11-38edd4dd406f","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:26Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"630448c8-38eb-4c91-b3c8-8f16692b65c4","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:26Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-pending-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-pending-sub","reconcileID":"630448c8-38eb-4c91-b3c8-8f16692b65c4","error":"failed to revoke API keys for subscription e2e-apikey-pending-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:28Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"8a3f5175-2d30-4a5c-85d8-d5c70e94788b","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:34:29Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"6609716e-f6ca-4ebc-a86f-9b6f93f80f5b","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:29Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"6609716e-f6ca-4ebc-a86f-9b6f93f80f5b","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:32Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"2a83cb71-2ee9-44d6-8649-eb3e02134e6c","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:32Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-active-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-active-sub","reconcileID":"2a83cb71-2ee9-44d6-8649-eb3e02134e6c","error":"failed to revoke API keys for subscription e2e-apikey-active-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:32Z","msg":"TokenRateLimitPolicy updated","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"1f0e6af4-dcfc-40d6-b6f1-0f657eea4d93","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1,"subscriptions":["models-as-a-service/premium-simulator-subscription"]}
{"level":"error","ts":"2026-07-03T19:34:35Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"fbc7fe74-de06-4e71-8bc8-c3fb7c5f2e71","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:35Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"fbc7fe74-de06-4e71-8bc8-c3fb7c5f2e71","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:38Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"ff717c51-dff4-4263-936a-fd9705be21b2","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:38Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"ff717c51-dff4-4263-936a-fd9705be21b2","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:41Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"3303b00f-a498-4068-9d6a-f929aab8585e","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:41Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"3303b00f-a498-4068-9d6a-f929aab8585e","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:44Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"88cbc9b3-577d-4eb5-8e2b-bfdbe6c3246d","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:44Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-degraded-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-degraded-sub","reconcileID":"88cbc9b3-577d-4eb5-8e2b-bfdbe6c3246d","error":"failed to revoke API keys for subscription e2e-apikey-degraded-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:47Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"e3a73651-f691-4696-a254-d304ac1849af","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:47Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"e3a73651-f691-4696-a254-d304ac1849af","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:34:49Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:34:50Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a8f13ef1-d2d7-446f-99e5-ac384653a3c7","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:50Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"a8f13ef1-d2d7-446f-99e5-ac384653a3c7","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:50Z","msg":"TokenRateLimitPolicy unchanged, skipping update","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"premium-simulator-subscription","reconcileID":"f6a54ff1-4921-4a6f-8eb0-f13b2d9ee89e","MaaSSubscription":{"name":"premium-simulator-subscription","namespace":"models-as-a-service"},"name":"maas-trlp-premium-simulated-simulated-premium","model":"llm/premium-simulated-simulated-premium","subscriptionCount":1}
{"level":"error","ts":"2026-07-03T19:34:53Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"db62a569-da59-4b51-9c04-806d618616bb","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:53Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"db62a569-da59-4b51-9c04-806d618616bb","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:56Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"a7e91c05-aa8a-4fab-b6e6-5ae5bc51de44","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:56Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-failed-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-failed-sub","reconcileID":"a7e91c05-aa8a-4fab-b6e6-5ae5bc51de44","error":"failed to revoke API keys for subscription e2e-apikey-failed-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:34:58Z","msg":"Deleting model group AuthPolicy so remaining policies can rebuild it","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-auth","reconcileID":"b579fd07-d335-4324-b7f9-3a533781e9ee","MaaSAuthPolicy":{"name":"e2e-ordering-auth","namespace":"models-as-a-service"},"model":"llm/premium-simulated-simulated-premium"}
{"level":"info","ts":"2026-07-03T19:34:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"1e3f13c4-4625-4f33-a7d9-0354af2817cb","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:34:58Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"namespace":"llm","name":"premium-simulated-simulated-premium","reconcileID":"ebf46274-f12a-4db8-a896-bf87a75df79e","MaaSModelRef":{"name":"premium-simulated-simulated-premium","namespace":"llm"},"routeName":"premium-simulated-simulated-premium-kserve-route","namespace":"llm","llmisvcName":"premium-simulated-simulated-premium","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:34:59Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"53a0cab1-cce3-4f35-a144-4e7520239415","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:34:59Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"53a0cab1-cce3-4f35-a144-4e7520239415","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:02Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"97e25ec5-a68b-459d-a96c-2c3e8de6ceee","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:02Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-rate-limit-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rate-limit-test-subscription","reconcileID":"97e25ec5-a68b-459d-a96c-2c3e8de6ceee","error":"failed to revoke API keys for subscription e2e-rate-limit-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:05Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"be1dcacf-7fff-477f-9287-8e183e2d40db","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:05Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-neg-ghost-sub-c537128a","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-neg-ghost-sub-c537128a","reconcileID":"be1dcacf-7fff-477f-9287-8e183e2d40db","error":"failed to revoke API keys for subscription e2e-neg-ghost-sub-c537128a after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:35:08Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"6c6bdf9d-669e-4bf8-9dc3-53f23799abec","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"info","ts":"2026-07-03T19:35:08Z","logger":"TenantIdentifierFor","msg":"Using default AITenant legacy resource identifier (empty string)","tenant":"models-as-a-service/default-tenant","labels":{"ai-gateway.opendatahub.io/tenant":"models-as-a-service","app.kubernetes.io/managed-by":"maas-controller","app.kubernetes.io/part-of":"modelsasservice","maas.opendatahub.io/managed-by-aitenant":"true","maas.opendatahub.io/tenant-name":"models-as-a-service","maas.opendatahub.io/tenant-namespace":"models-as-a-service"},"tenantName":"models-as-a-service"}
{"level":"info","ts":"2026-07-03T19:35:08Z","msg":"reconcileGatewayAuthPolicy entered","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"c188e249-9a56-491e-bc44-ae9721e94902","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"gatewayNamespace":"openshift-ingress","gatewayName":"maas-default-gateway","tenantID":"","xAPIKeyEnabled":false}
{"level":"info","ts":"2026-07-03T19:35:08Z","msg":"unknown field \"spec.targetRef.namespace\"","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"c188e249-9a56-491e-bc44-ae9721e94902"}
{"level":"info","ts":"2026-07-03T19:35:08Z","msg":"gateway AuthPolicy updated","controller":"maasauthpolicy","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSAuthPolicy","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-access","reconcileID":"c188e249-9a56-491e-bc44-ae9721e94902","MaaSAuthPolicy":{"name":"simulator-access","namespace":"models-as-a-service"},"name":"maas-gateway-auth","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-07-03T19:35:08Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"88f56e32-4b6c-4d41-81c9-4b1b37962624","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:08Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"simulator-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"simulator-subscription","reconcileID":"88f56e32-4b6c-4d41-81c9-4b1b37962624","error":"failed to revoke API keys for subscription simulator-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:11Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"1d885f83-e825-402b-82e1-e6c03ec4dc33","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:11Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-filter-sub-b-b16c7d97","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-filter-sub-b-b16c7d97","reconcileID":"1d885f83-e825-402b-82e1-e6c03ec4dc33","error":"failed to revoke API keys for subscription e2e-filter-sub-b-b16c7d97 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:35:12Z","msg":"HTTPRoute validated for LLMInferenceService","controller":"maasmodelref","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSModelRef","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"namespace":"llm","name":"facebook-opt-125m-simulated","reconcileID":"b168f7ca-907d-4b54-a40e-7ef25ae10fea","MaaSModelRef":{"name":"facebook-opt-125m-simulated","namespace":"llm"},"routeName":"facebook-opt-125m-simulated-kserve-route","namespace":"llm","llmisvcName":"facebook-opt-125m-simulated","gateway":"openshift-ingress/maas-default-gateway","hostnames":[]}
{"level":"error","ts":"2026-07-03T19:35:14Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"1cc592c5-f5d9-4882-ac00-f685ee8848a1","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:14Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-ordering-sub","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-ordering-sub","reconcileID":"1cc592c5-f5d9-4882-ac00-f685ee8848a1","error":"failed to revoke API keys for subscription e2e-ordering-sub after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:17Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"855bcfb2-6460-4c9f-ac39-6f9ffd385fa2","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:17Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-models-exempt-test-subscription","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-models-exempt-test-subscription","reconcileID":"855bcfb2-6460-4c9f-ac39-6f9ffd385fa2","error":"failed to revoke API keys for subscription e2e-models-exempt-test-subscription after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-07-03T19:35:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"ai-tenants","purpose":"aitenant","phase":"Active"}
{"level":"info","ts":"2026-07-03T19:35:19Z","logger":"setup","msg":"managed namespace already exists","namespace":"models-as-a-service","purpose":"subscription","phase":"Active"}
{"level":"error","ts":"2026-07-03T19:35:20Z","msg":"failed to revoke API keys for subscription, will retry","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"2e94f016-6728-46f3-87ea-98ae4e137ae2","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).handleDeletion\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:857\ngithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas.(*MaaSSubscriptionReconciler).Reconcile\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/pkg/controller/maas/maassubscription_controller.go:357\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-07-03T19:35:20Z","msg":"Reconciler error","controller":"maassubscription","controllerGroup":"maas.opendatahub.io","controllerKind":"MaaSSubscription","MaaSSubscription":{"name":"e2e-apikey-sub-binding-4a58c111","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-apikey-sub-binding-4a58c111","reconcileID":"2e94f016-6728-46f3-87ea-98ae4e137ae2","error":"failed to revoke API keys for subscription e2e-apikey-sub-binding-4a58c111 after retries: HTTP request failed: Post \"https://maas-api.models-as-a-service.svc.cluster.local:8443/internal/v1/api-keys/revoke-for-subscription\": dial tcp: lookup maas-api.models-as-a-service.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.5/pkg/internal/controller/controller.go:296"}