{"level":"info","ts":"2026-11-02T14:52:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e16d1c0e"},"tenantID":"e2e-mt-e16d1c0e","namespace":"openshift-ingress","gatewayName":"e2e-mt-e16d1c0e"}
{"level":"info","ts":"2026-11-02T14:52:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:52:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:52:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:52:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:52:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:52:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:52:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:52:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e16d1c0e"},"tenantID":"e2e-mt-e16d1c0e","namespace":"openshift-ingress","gatewayName":"e2e-mt-e16d1c0e"}
{"level":"error","ts":"2026-11-02T14:52:31Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e16d1c0e"},"error":"apply NetworkPolicy ai-tenant-e2e-mt-e16d1c0e/payload-processing-e2e-mt-e16d1c0e: networkpolicies.networking.k8s.io \"payload-processing-e2e-mt-e16d1c0e\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-e16d1c0e because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:52:31Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e16d1c0e"},"namespace":"ai-tenant-e2e-mt-e16d1c0e","name":"default-tenant","reconcileID":"bc04c4d5-e1da-426e-ac3e-e3d145a9889b","error":"apply: apply NetworkPolicy ai-tenant-e2e-mt-e16d1c0e/payload-processing-e2e-mt-e16d1c0e: networkpolicies.networking.k8s.io \"payload-processing-e2e-mt-e16d1c0e\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-e16d1c0e because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:52:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e16d1c0e"},"tenantID":"e2e-mt-e16d1c0e","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:52:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:52:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:52:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:52:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:52:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:52:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-26b508"},"tenantID":"e2e-extmod-a-26b508","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-26b508"}
{"level":"info","ts":"2026-11-02T14:52:40Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-a4809a"},"tenantID":"e2e-shared-a-w3-a4809a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:52:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:52:41Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-a4809a"},"tenantID":"e2e-shared-a-w3-a4809a","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:52:41Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-a4809a"},"namespace":"ai-tenant-e2e-shared-a-w3-a4809a","name":"default-tenant","reconcileID":"a2ba3b70-e554-4759-802d-3453d3cd237e","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:52:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:52:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ee3eca"},"tenantID":"e2e-extmod-b-ee3eca","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-ee3eca"}
{"level":"info","ts":"2026-11-02T14:52:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:52:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-21baa7"}
{"level":"info","ts":"2026-11-02T14:52:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-21baa7"}
{"level":"info","ts":"2026-11-02T14:52:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:52:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:52:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:52:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:52:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:52:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:52:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:52:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:52:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:52:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:52:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:53:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:53:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-21baa7"}
{"level":"info","ts":"2026-11-02T14:53:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:53:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:53:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:53:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:53:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"tenantID":"e2e-shared-a-w4-2cddf2","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-2cddf2"}
{"level":"info","ts":"2026-11-02T14:53:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"tenantID":"e2e-shared-a-w4-2cddf2","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-2cddf2"}
{"level":"info","ts":"2026-11-02T14:53:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:53:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:53:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-662e3e"},"tenantID":"e2e-shared-b-w3-662e3e","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-662e3e"}
{"level":"error","ts":"2026-11-02T14:53:12Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-662e3e"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w3-662e3e/payload-processing-e2e-shared-b-w3-662e3e: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-662e3e\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-662e3e because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:53:12Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-662e3e"},"namespace":"ai-tenant-e2e-shared-b-w3-662e3e","name":"default-tenant","reconcileID":"cadedc71-9109-4b48-95cd-451385372197","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w3-662e3e/payload-processing-e2e-shared-b-w3-662e3e: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-662e3e\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-662e3e because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:53:13Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-662e3e"},"tenantID":"e2e-shared-b-w3-662e3e","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:53:15Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-2daae4"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w0-2daae4/payload-processing-e2e-shared-b-w0-2daae4: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w0-2daae4\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w0-2daae4 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:53:15Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-2daae4"},"namespace":"ai-tenant-e2e-shared-b-w0-2daae4","name":"default-tenant","reconcileID":"3cdf3da8-1eea-4b71-b7e7-ee47d2f97e4c","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w0-2daae4/payload-processing-e2e-shared-b-w0-2daae4: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w0-2daae4\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w0-2daae4 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:53:16Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-2daae4"},"tenantID":"e2e-shared-b-w0-2daae4","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:53:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:53:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:53:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:53:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:53:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:53:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:53:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"tenantID":"e2e-shared-a-w4-2cddf2","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-2cddf2"}
{"level":"info","ts":"2026-11-02T14:53:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-0d1ca7"}
{"level":"info","ts":"2026-11-02T14:53:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-0d1ca7"}
{"level":"info","ts":"2026-11-02T14:53:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:53:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:53:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:53:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:53:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:53:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:53:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-0d1ca7"}
{"level":"info","ts":"2026-11-02T14:53:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:53:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:53:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:53:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:53:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"tenantID":"e2e-shared-b-w4-4c2811","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-4c2811"}
{"level":"info","ts":"2026-11-02T14:53:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"tenantID":"e2e-shared-b-w4-4c2811","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-4c2811"}
{"level":"info","ts":"2026-11-02T14:53:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:53:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:53:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:53:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:53:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:53:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:53:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:53:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:53:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"tenantID":"e2e-shared-b-w4-4c2811","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-4c2811"}
{"level":"info","ts":"2026-11-02T14:53:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-1a562c"},"tenantID":"e2e-worker-w0-1a562c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-1a562c"}
{"level":"info","ts":"2026-11-02T14:53:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-1a562c"},"tenantID":"e2e-worker-w0-1a562c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-1a562c"}
{"level":"info","ts":"2026-11-02T14:53:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:53:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:53:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:53:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:53:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:53:59Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-1a562c"},"tenantID":"e2e-worker-w0-1a562c","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:53:59Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-1a562c"},"tenantID":"e2e-worker-w0-1a562c","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:53:59Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-1a562c"},"namespace":"ai-tenant-e2e-worker-w0-1a562c","name":"default-tenant","reconcileID":"4ec2ea55-c651-41af-ac47-aad14096b1e9","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:54:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:54:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:54:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:54:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:54:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:54:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-f30173"},"tenantID":"e2e-worker-w3-f30173","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-f30173"}
{"level":"info","ts":"2026-11-02T14:54:09Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-f30173"},"tenantID":"e2e-worker-w3-f30173","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:54:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:54:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:54:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:54:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:54:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:54:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:54:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:54:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:54:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:54:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:54:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:54:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:54:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:54:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:54:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T14:54:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:54:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:54:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:54:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:54:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:54:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:54:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:54:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:54:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:54:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:54:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:54:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:54:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:54:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:54:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:54:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:54:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:54:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:54:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:54:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:54:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:54:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:54:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:55:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:55:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:55:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:55:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:55:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:55:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:55:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:55:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:55:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:55:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ce0f55"},"tenantID":"e2e-extmod-a-ce0f55","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-ce0f55"}
{"level":"info","ts":"2026-11-02T14:55:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-66bc6d"},"tenantID":"e2e-extmod-b-66bc6d","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-66bc6d"}
{"level":"info","ts":"2026-11-02T14:55:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"tenantID":"e2e-shared-a-w2-960a68","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-960a68"}
{"level":"info","ts":"2026-11-02T14:55:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:55:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:55:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:55:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:55:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:55:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:55:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:55:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:55:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:55:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:55:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:55:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:55:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:55:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:55:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:55:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:55:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:55:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-ad42e0"},"tenantID":"e2e-worker-w4-ad42e0","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-ad42e0"}
{"level":"info","ts":"2026-11-02T14:55:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:55:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:55:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:55:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:55:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-92f249"},"tenantID":"e2e-worker-w1-92f249","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-92f249"}
{"level":"info","ts":"2026-11-02T14:55:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:55:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:55:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:55:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:55:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:55:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:55:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:55:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-fcefcc"},"tenantID":"e2e-shared-b-w2-fcefcc","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-fcefcc"}
{"level":"info","ts":"2026-11-02T14:55:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:55:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:55:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:55:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:55:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:56:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:56:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:56:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:56:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-21baa7"}
{"level":"info","ts":"2026-11-02T14:56:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-21baa7"}
{"level":"info","ts":"2026-11-02T14:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:56:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:56:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:56:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:56:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:56:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"tenantID":"e2e-shared-a-w4-2cddf2","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-2cddf2"}
{"level":"info","ts":"2026-11-02T14:56:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"tenantID":"e2e-shared-a-w4-2cddf2","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-2cddf2"}
{"level":"info","ts":"2026-11-02T14:56:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"tenantID":"e2e-shared-b-w4-4c2811","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-4c2811"}
{"level":"info","ts":"2026-11-02T14:56:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"tenantID":"e2e-shared-b-w4-4c2811","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-4c2811"}
{"level":"info","ts":"2026-11-02T14:56:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:56:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-5f98a6"},"tenantID":"e2e-extmod-a-5f98a6","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-5f98a6"}
{"level":"info","ts":"2026-11-02T14:56:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-0d1ca7"}
{"level":"info","ts":"2026-11-02T14:56:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-0d1ca7"}
{"level":"info","ts":"2026-11-02T14:56:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a02c35"},"tenantID":"e2e-extmod-b-a02c35","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a02c35"}
{"level":"info","ts":"2026-11-02T14:56:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:56:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:56:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:56:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:56:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:56:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:56:22Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-26b508"},"tenantID":"e2e-extmod-a-26b508","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:56:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ce0f55"},"tenantID":"e2e-extmod-a-ce0f55","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-ce0f55"}
{"level":"info","ts":"2026-11-02T14:56:25Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ce0f55"},"tenantID":"e2e-extmod-a-ce0f55","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:56:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:56:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:56:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:56:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:56:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:56:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:56:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:56:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:56:39Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-66bc6d"}}
{"level":"info","ts":"2026-11-02T14:56:39Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-66bc6d"},"tenantID":"e2e-extmod-b-66bc6d","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:56:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:56:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:56:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:56:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:56:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:56:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:56:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:56:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:56:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:56:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:56:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:56:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:56:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:56:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"tenantID":"e2e-shared-a-w2-960a68","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-960a68"}
{"level":"info","ts":"2026-11-02T14:56:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-5f98a6"},"tenantID":"e2e-extmod-a-5f98a6","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-5f98a6"}
{"level":"info","ts":"2026-11-02T14:56:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"tenantID":"e2e-shared-a-w2-960a68","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-960a68"}
{"level":"info","ts":"2026-11-02T14:56:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-fcefcc"},"tenantID":"e2e-shared-b-w2-fcefcc","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-fcefcc"}
{"level":"error","ts":"2026-11-02T14:56:55Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ee3eca"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-b-ee3eca/payload-processing-e2e-extmod-b-ee3eca: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-ee3eca\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-ee3eca because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:56:55Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ee3eca"},"namespace":"ai-tenant-e2e-extmod-b-ee3eca","name":"default-tenant","reconcileID":"49830de2-65e2-46ea-921c-fda139b16753","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-b-ee3eca/payload-processing-e2e-extmod-b-ee3eca: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-ee3eca\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-ee3eca because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:56:55Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ee3eca"}}
{"level":"info","ts":"2026-11-02T14:56:56Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ee3eca"},"tenantID":"e2e-extmod-b-ee3eca","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:56:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"}}
{"level":"info","ts":"2026-11-02T14:56:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-5f98a6"},"tenantID":"e2e-extmod-a-5f98a6","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-5f98a6"}
{"level":"info","ts":"2026-11-02T14:57:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-5f98a6"},"tenantID":"e2e-extmod-a-5f98a6","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-5f98a6"}
{"level":"info","ts":"2026-11-02T14:57:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:57:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:57:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:57:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:57:00Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-5f98a6"}}
{"level":"info","ts":"2026-11-02T14:57:01Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-5f98a6"},"tenantID":"e2e-extmod-a-5f98a6","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:57:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"error","ts":"2026-11-02T14:57:06Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-47bb2fdc"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:57:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:57:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:57:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:57:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:57:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:57:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:57:14Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a02c35"}}
{"level":"info","ts":"2026-11-02T14:57:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a02c35"},"tenantID":"e2e-extmod-b-a02c35","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-0969b1"},"tenantID":"e2e-worker-w2-0969b1","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-0969b1"}
{"level":"info","ts":"2026-11-02T14:57:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:57:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:57:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:57:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:57:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:57:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:57:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-21baa7"}
{"level":"info","ts":"2026-11-02T14:57:23Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:24Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"tenantID":"e2e-shared-a-w1-21baa7","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:57:24Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-21baa7"},"namespace":"ai-tenant-e2e-shared-a-w1-21baa7","name":"default-tenant","reconcileID":"3e303373-69f4-4f45-a29a-678c90d19262","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:25Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w4-2cddf2/payload-processing-e2e-shared-a-w4-2cddf2: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w4-2cddf2\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w4-2cddf2 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:25Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"namespace":"ai-tenant-e2e-shared-a-w4-2cddf2","name":"default-tenant","reconcileID":"6e235610-3cdd-4f18-af23-b6fa53fbfe4c","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w4-2cddf2/payload-processing-e2e-shared-a-w4-2cddf2: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w4-2cddf2\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w4-2cddf2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:57:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-2cddf2"},"tenantID":"e2e-shared-a-w4-2cddf2","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:57:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:57:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:57:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:57:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:57:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"error","ts":"2026-11-02T14:57:36Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w2-960a68/payload-processing-e2e-shared-a-w2-960a68: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w2-960a68\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w2-960a68 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:36Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"namespace":"ai-tenant-e2e-shared-a-w2-960a68","name":"default-tenant","reconcileID":"c4344755-af83-4df3-b6e7-a64453e76614","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w2-960a68/payload-processing-e2e-shared-a-w2-960a68: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w2-960a68\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w2-960a68 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:37Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w2-960a68/payload-processing-e2e-shared-a-w2-960a68: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w2-960a68\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w2-960a68 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:37Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"namespace":"ai-tenant-e2e-shared-a-w2-960a68","name":"default-tenant","reconcileID":"b2b1bbad-1ea7-4c98-85f6-bc831931ab05","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w2-960a68/payload-processing-e2e-shared-a-w2-960a68: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w2-960a68\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w2-960a68 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:57:37Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"}}
{"level":"info","ts":"2026-11-02T14:57:38Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-960a68"},"tenantID":"e2e-shared-a-w2-960a68","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:57:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:57:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:57:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:57:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:57:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:57:45Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:46Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"tenantID":"e2e-shared-b-w1-0d1ca7","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:57:46Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7"},"namespace":"ai-tenant-e2e-shared-b-w1-0d1ca7","name":"default-tenant","reconcileID":"a412598c-cf48-494b-b64f-1b65549f5c29","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:49Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w4-4c2811/payload-processing-e2e-shared-b-w4-4c2811: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w4-4c2811\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w4-4c2811 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:49Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"namespace":"ai-tenant-e2e-shared-b-w4-4c2811","name":"default-tenant","reconcileID":"04f50f74-189b-4889-8783-8272dc2e1e0f","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w4-4c2811/payload-processing-e2e-shared-b-w4-4c2811: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w4-4c2811\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w4-4c2811 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:49Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w4-4c2811/payload-processing-e2e-shared-b-w4-4c2811: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w4-4c2811\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w4-4c2811 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:57:49Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"namespace":"ai-tenant-e2e-shared-b-w4-4c2811","name":"default-tenant","reconcileID":"8fc6cf34-84e5-45b8-8503-8fc4024c02b7","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w4-4c2811/payload-processing-e2e-shared-b-w4-4c2811: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w4-4c2811\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w4-4c2811 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:57:50Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-4c2811"},"tenantID":"e2e-shared-b-w4-4c2811","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:57:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:57:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"}}
{"level":"info","ts":"2026-11-02T14:57:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:57:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:57:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:57:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:58:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"error","ts":"2026-11-02T14:58:00Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-79d4ba83"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:58:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:58:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:58:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:58:08Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-92f249"},"tenantID":"e2e-worker-w1-92f249","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:58:08Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-92f249"},"tenantID":"e2e-worker-w1-92f249","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:58:08Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-92f249"},"namespace":"ai-tenant-e2e-worker-w1-92f249","name":"default-tenant","reconcileID":"43e1b059-35b4-4a76-86ff-85d03953da4e","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-fcefcc"},"tenantID":"e2e-shared-b-w2-fcefcc","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-fcefcc"}
{"level":"error","ts":"2026-11-02T14:58:09Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-ad42e0"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w4-ad42e0/payload-processing-e2e-worker-w4-ad42e0: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-ad42e0\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-ad42e0 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:58:09Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-ad42e0"},"namespace":"ai-tenant-e2e-worker-w4-ad42e0","name":"default-tenant","reconcileID":"738962e8-7b0c-4ee6-a81c-68226090ad65","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w4-ad42e0/payload-processing-e2e-worker-w4-ad42e0: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-ad42e0\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-ad42e0 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:10Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-fcefcc"},"tenantID":"e2e-shared-b-w2-fcefcc","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-ad42e0"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w4-ad42e0/payload-processing-e2e-worker-w4-ad42e0: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-ad42e0\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-ad42e0 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:58:11Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-ad42e0"},"namespace":"ai-tenant-e2e-worker-w4-ad42e0","name":"default-tenant","reconcileID":"abf4a746-1b12-499a-8d92-95885884127b","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w4-ad42e0/payload-processing-e2e-worker-w4-ad42e0: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-ad42e0\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-ad42e0 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"}}
{"level":"info","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-ad42e0"},"tenantID":"e2e-worker-w4-ad42e0","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:58:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:58:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"error","ts":"2026-11-02T14:58:21Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-a95b1130"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:58:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:58:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"error","ts":"2026-11-02T14:58:31Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-0969b1"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w2-0969b1/payload-processing-e2e-worker-w2-0969b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-0969b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-0969b1 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:58:31Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-0969b1"},"namespace":"ai-tenant-e2e-worker-w2-0969b1","name":"default-tenant","reconcileID":"7203fcd5-2a67-404f-8308-1693170622b1","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w2-0969b1/payload-processing-e2e-worker-w2-0969b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-0969b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-0969b1 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:58:31Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-0969b1"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w2-0969b1/payload-processing-e2e-worker-w2-0969b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-0969b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-0969b1 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T14:58:31Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-0969b1"},"namespace":"ai-tenant-e2e-worker-w2-0969b1","name":"default-tenant","reconcileID":"19a1e363-d0e4-4336-bcb9-c9a62d20ade0","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w2-0969b1/payload-processing-e2e-worker-w2-0969b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-0969b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-0969b1 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:58:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:58:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-0969b1"},"tenantID":"e2e-worker-w2-0969b1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:58:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:58:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:58:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"}}
{"level":"info","ts":"2026-11-02T14:58:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:58:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:58:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"error","ts":"2026-11-02T14:58:51Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-742533f1"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:58:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:58:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:58:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:59:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:59:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:59:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:59:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:59:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:59:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:59:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:59:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:59:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:59:25Z","logger":"tenant","msg":"waiting for the legacy IPP cleanup to finish before applying praxis-extproc","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"}}
{"level":"info","ts":"2026-11-02T14:59:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"},"tenantID":"e2e-ait-fcad4d9a","namespace":"openshift-ingress","gatewayName":"e2e-ait-fcad4d9a"}
{"level":"info","ts":"2026-11-02T14:59:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"},"tenantID":"e2e-ait-fcad4d9a","namespace":"openshift-ingress","gatewayName":"e2e-ait-fcad4d9a"}
{"level":"info","ts":"2026-11-02T14:59:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T14:59:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"},"tenantID":"e2e-ait-fcad4d9a","namespace":"openshift-ingress","gatewayName":"e2e-ait-fcad4d9a"}
{"level":"info","ts":"2026-11-02T14:59:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"}}
{"level":"info","ts":"2026-11-02T14:59:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:59:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:59:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"},"tenantID":"e2e-ait-fcad4d9a","namespace":"openshift-ingress","gatewayName":"e2e-ait-fcad4d9a"}
{"level":"info","ts":"2026-11-02T14:59:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"},"tenantID":"e2e-ait-fcad4d9a","namespace":"openshift-ingress","gatewayName":"e2e-ait-fcad4d9a"}
{"level":"info","ts":"2026-11-02T14:59:40Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"}}
{"level":"info","ts":"2026-11-02T14:59:40Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"}}
{"level":"error","ts":"2026-11-02T14:59:41Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-bdefae96"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T14:59:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:59:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T14:59:50Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"}}
{"level":"info","ts":"2026-11-02T14:59:51Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-fcad4d9a"},"tenantID":"e2e-ait-fcad4d9a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T14:59:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T14:59:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:00:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T15:00:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:00:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T15:00:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:00:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T15:00:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:00:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T15:00:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:00:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T15:00:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:00:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"info","ts":"2026-11-02T15:00:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"}}
{"level":"info","ts":"2026-11-02T15:01:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"error","ts":"2026-11-02T15:01:02Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-e16d1c0e"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:01:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"}}
{"level":"error","ts":"2026-11-02T15:01:22Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-65daea46"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:02:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:02:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:02:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:02:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:03:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:03:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:03:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:03:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-11-02T15:03:29Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"namespace":"ai-tenant-e2e-worker-single-main-6e5c6f","name":"default-tenant","reconcileID":"d6aaacd3-547a-45e2-b611-427ac8fad273","error":"ensure finalizer: Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:03:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:03:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:03:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:03:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:03:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:04:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:04:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:04:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"error","ts":"2026-11-02T15:04:36Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-single-main-6e5c6f/payload-processing-e2e-worker-single-main-6e5c6f: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-single-main-6e5c6f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-single-main-6e5c6f because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-02T15:04:36Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"namespace":"ai-tenant-e2e-worker-single-main-6e5c6f","name":"default-tenant","reconcileID":"c56d7685-7d99-4f52-bfb1-f4164db04b7b","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-single-main-6e5c6f/payload-processing-e2e-worker-single-main-6e5c6f: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-single-main-6e5c6f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-single-main-6e5c6f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:04:37Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T15:05:10Z","logger":"tenant","msg":"waiting for the legacy IPP cleanup to finish before applying praxis-extproc","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:05:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-6e5c6f"}
{"level":"info","ts":"2026-11-02T15:05:15Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:15Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:16Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:19Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:23Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:33Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"}}
{"level":"info","ts":"2026-11-02T15:05:33Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-6e5c6f"},"tenantID":"e2e-worker-single-main-6e5c6f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T15:07:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:07:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:07:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:08:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:08:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:08:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:08:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-11-02T15:10:07Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rotation-external-model","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rotation-external-model","reconcileID":"c0a13521-240a-4a8d-a1ac-de63ab0440bf","error":"remove ExternalModel cleanup finalizer: externalmodels.inference.opendatahub.io \"e2e-rotation-external-model\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:10:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:10:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:13:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:13:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:13:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:13:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:14:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:14:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:14:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-11-02T15:14:39Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"namespace":"ai-tenant-e2e-worker-single-main-437128","name":"default-tenant","reconcileID":"16da0ff5-4c4c-442a-9ac1-23c4f1e10ac2","error":"ensure finalizer: Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:14:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:14:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:14:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:14:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:14:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:15:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:15:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:15:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:15:44Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:15:45Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-02T15:16:19Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"namespace":"ai-tenant-e2e-worker-single-main-437128","name":"default-tenant","reconcileID":"ecba9d3b-800b-4dff-8cc4-6162ca6e2f75","error":"ensure finalizer: Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-02T15:16:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:16:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-437128"}
{"level":"info","ts":"2026-11-02T15:16:21Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:16:21Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:16:22Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:16:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:16:30Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:16:40Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"}}
{"level":"info","ts":"2026-11-02T15:16:40Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-437128"},"tenantID":"e2e-worker-single-main-437128","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-02T15:16:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:16:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:16:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:09Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-11-02T15:19:09Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-11-02T15:19:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:10Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-11-02T15:19:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:15Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-11-02T15:19:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:38Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-11-02T15:19:38Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-11-02T15:19:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:19:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:20:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:20:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:20:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:20:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-02T15:21:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}