{"level":"info","ts":"2026-11-03T01:07:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:07:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:07:57Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:08:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:08:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:08:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:08:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:08:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:08:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:08:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:08:07Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:08:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:08:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:08:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:08:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:08:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:08:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:08:17Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:08:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:08:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:08:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:08:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:08:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:08:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:08:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:08:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:08:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:08:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:08:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:08:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:08:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:08:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:08:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:08:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:08:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:08:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:08:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-7ef1b6"}
{"level":"info","ts":"2026-11-03T01:08:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:08:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:08:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-7ef1b6"}
{"level":"info","ts":"2026-11-03T01:08:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:08:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:08:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:09:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:09:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:09:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:09:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:09:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:09:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"error","ts":"2026-11-03T01:09:13Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d87781"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-d87781/payload-processing-e2e-extmod-a-d87781: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-d87781\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-d87781 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:09:13Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d87781"},"namespace":"ai-tenant-e2e-extmod-a-d87781","name":"default-tenant","reconcileID":"c6a81ed8-1a64-4d89-ba57-cada61675bce","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-d87781/payload-processing-e2e-extmod-a-d87781: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-d87781\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-d87781 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:09:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:09:14Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d87781"}}
{"level":"info","ts":"2026-11-03T01:09:14Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d87781"},"tenantID":"e2e-extmod-a-d87781","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:09:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-7ef1b6"}
{"level":"info","ts":"2026-11-03T01:09:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:09:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:09:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:09:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:09:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:09:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:09:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:09:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:09:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:09:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:09:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:09:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:09:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:09:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:09:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:09:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:09:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:09:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:09:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:09:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:09:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:09:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"},"tenantID":"e2e-extmod-b-d4d8c2","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-d4d8c2"}
{"level":"info","ts":"2026-11-03T01:09:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:09:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:09:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:48Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:09:48Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:09:48Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:09:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:09:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:09:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:09:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:09:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:09:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:09:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:09:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:10:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:10:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-955dc3"},"tenantID":"e2e-extmod-a-955dc3","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-955dc3"}
{"level":"info","ts":"2026-11-03T01:10:02Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-955dc3"},"tenantID":"e2e-extmod-a-955dc3","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:10:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:10:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:10:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:10:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:10:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:10:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:10:10Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"}}
{"level":"info","ts":"2026-11-03T01:10:10Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"},"tenantID":"e2e-extmod-b-d4d8c2","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:10:10Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"},"namespace":"ai-tenant-e2e-extmod-b-d4d8c2","name":"default-tenant","reconcileID":"9e8bf12f-850b-4e09-96c0-bb815d84d4e7","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:10:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:10:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"},"tenantID":"e2e-extmod-b-d4d8c2","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:10:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"},"tenantID":"e2e-extmod-b-d4d8c2","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:10:11Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-d4d8c2"},"namespace":"ai-tenant-e2e-extmod-b-d4d8c2","name":"default-tenant","reconcileID":"0562fb68-7fc0-4825-82ad-ee8ccc464091","error":"maastenantconfigs.maas.opendatahub.io \"default-tenant\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:10:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:10:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:10:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:10:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c738e0"},"tenantID":"e2e-extmod-a-c738e0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-c738e0"}
{"level":"info","ts":"2026-11-03T01:10:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:10:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:10:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:10:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:10:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:10:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"error","ts":"2026-11-03T01:10:25Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-359f71"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w3-359f71/payload-processing-e2e-shared-a-w3-359f71: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w3-359f71\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w3-359f71 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:10:25Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-359f71"},"namespace":"ai-tenant-e2e-shared-a-w3-359f71","name":"default-tenant","reconcileID":"92f6d52a-26e7-489f-b1e4-b59a35fc7dcc","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w3-359f71/payload-processing-e2e-shared-a-w3-359f71: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w3-359f71\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w3-359f71 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:10:25Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-359f71"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w3-359f71/payload-processing-e2e-shared-a-w3-359f71: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w3-359f71\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w3-359f71 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:10:25Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-359f71"},"namespace":"ai-tenant-e2e-shared-a-w3-359f71","name":"default-tenant","reconcileID":"15bf2d87-01a6-48bd-9df0-af7a446c0f06","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w3-359f71/payload-processing-e2e-shared-a-w3-359f71: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w3-359f71\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w3-359f71 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:10:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:10:26Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-359f71"}}
{"level":"info","ts":"2026-11-03T01:10:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-359f71"},"tenantID":"e2e-shared-a-w3-359f71","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:10:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:10:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:10:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:10:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:10:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:10:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:10:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:10:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:10:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:10:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:10:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:10:41Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-909268"},"tenantID":"e2e-extmod-b-909268","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:10:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:10:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:10:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:10:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-0db47e"},"tenantID":"e2e-extmod-b-0db47e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-0db47e"}
{"level":"error","ts":"2026-11-03T01:10:47Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-2858fb"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w3-2858fb/payload-processing-e2e-shared-b-w3-2858fb: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-2858fb\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-2858fb because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:10:47Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-2858fb"},"namespace":"ai-tenant-e2e-shared-b-w3-2858fb","name":"default-tenant","reconcileID":"03b03258-2368-48a8-a1d8-282ce46eb64f","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w3-2858fb/payload-processing-e2e-shared-b-w3-2858fb: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-2858fb\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-2858fb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:10:48Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-2858fb"},"tenantID":"e2e-shared-b-w3-2858fb","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:10:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:10:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:10:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:10:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:10:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:10:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:11:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:11:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:11:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:11:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:11:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e62935"},"tenantID":"e2e-shared-a-w0-e62935","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e62935"}
{"level":"info","ts":"2026-11-03T01:11:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e62935"},"tenantID":"e2e-shared-a-w0-e62935","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e62935"}
{"level":"info","ts":"2026-11-03T01:11:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:11:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:11:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:11:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:11:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress","gatewayName":"e2e-mt-c6e808d7"}
{"level":"info","ts":"2026-11-03T01:11:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:11:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:11:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:11:19Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:11:19Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"tenantID":"e2e-mt-c6e808d7","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:11:19Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c6e808d7"},"namespace":"ai-tenant-e2e-mt-c6e808d7","name":"default-tenant","reconcileID":"5027155d-63c6-4d55-a5f4-94eeda3f2e6e","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:11:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:11:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:11:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:11:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:11:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:11:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:11:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:11:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:11:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:11:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:11:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:11:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:11:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:11:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e62935"},"tenantID":"e2e-shared-a-w0-e62935","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:11:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e62935"},"tenantID":"e2e-shared-a-w0-e62935","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:11:32Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e62935"},"namespace":"ai-tenant-e2e-shared-a-w0-e62935","name":"default-tenant","reconcileID":"6f0196ca-c2c9-43e3-bf74-533f5791f83c","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:11:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:11:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:11:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:11:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:11:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:11:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:11:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:11:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-7ef1b6"}
{"level":"info","ts":"2026-11-03T01:11:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:11:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:11:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:11:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:11:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:11:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:11:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:11:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-99bdb8"},"tenantID":"e2e-worker-w4-99bdb8","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-99bdb8"}
{"level":"info","ts":"2026-11-03T01:11:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-8e90c5"},"tenantID":"e2e-worker-w3-8e90c5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-8e90c5"}
{"level":"info","ts":"2026-11-03T01:11:47Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-8e90c5"},"tenantID":"e2e-worker-w3-8e90c5","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:11:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:11:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"tenantID":"e2e-worker-w0-fc9316","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-fc9316"}
{"level":"info","ts":"2026-11-03T01:11:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:11:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:11:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:11:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-4c61c8"},"tenantID":"e2e-worker-w2-4c61c8","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-4c61c8"}
{"level":"info","ts":"2026-11-03T01:11:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:11:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:11:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:11:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-a07a90"},"tenantID":"e2e-shared-b-w0-a07a90","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-a07a90"}
{"level":"info","ts":"2026-11-03T01:11:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-a07a90"},"tenantID":"e2e-shared-b-w0-a07a90","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-a07a90"}
{"level":"info","ts":"2026-11-03T01:11:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:11:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:11:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:12:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:12:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:12:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:12:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:12:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-b85343"},"tenantID":"e2e-worker-w1-b85343","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-b85343"}
{"level":"info","ts":"2026-11-03T01:12:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:12:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:12:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:12:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:12:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:12:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:12:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:12:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:12:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:12:11Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-a07a90"}}
{"level":"info","ts":"2026-11-03T01:12:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-a07a90"},"tenantID":"e2e-shared-b-w0-a07a90","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:12:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T01:12:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:12:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:12:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:12:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:12:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:12:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:12:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:12:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:12:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:12:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:12:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:12:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:12:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:12:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:12:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:12:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:12:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:12:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:12:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ac338d"},"tenantID":"e2e-extmod-b-ac338d","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-ac338d"}
{"level":"info","ts":"2026-11-03T01:12:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"tenantID":"e2e-shared-b-w1-5c109b","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-5c109b"}
{"level":"info","ts":"2026-11-03T01:12:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:12:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:12:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:12:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:12:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-850433"},"tenantID":"e2e-extmod-a-850433","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-850433"}
{"level":"info","ts":"2026-11-03T01:12:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-065de0"}
{"level":"info","ts":"2026-11-03T01:12:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d9e067"},"tenantID":"e2e-extmod-a-d9e067","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-d9e067"}
{"level":"info","ts":"2026-11-03T01:12:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fdacdb"},"tenantID":"e2e-extmod-b-fdacdb","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-fdacdb"}
{"level":"info","ts":"2026-11-03T01:12:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:12:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:12:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:12:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:12:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:12:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-7ef1b6"}
{"level":"info","ts":"2026-11-03T01:12:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-7ef1b6"}
{"level":"info","ts":"2026-11-03T01:12:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:12:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:12:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:12:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:12:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:12:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:12:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d9e067"},"tenantID":"e2e-extmod-a-d9e067","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-d9e067"}
{"level":"info","ts":"2026-11-03T01:12:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d9e067"},"tenantID":"e2e-extmod-a-d9e067","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-d9e067"}
{"level":"info","ts":"2026-11-03T01:12:53Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-d9e067"},"tenantID":"e2e-extmod-a-d9e067","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:12:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:12:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"tenantID":"e2e-shared-b-w1-5c109b","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-5c109b"}
{"level":"info","ts":"2026-11-03T01:12:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:12:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"}}
{"level":"info","ts":"2026-11-03T01:12:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:13:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:13:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:13:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"error","ts":"2026-11-03T01:13:02Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w0-fc9316/payload-processing-e2e-worker-w0-fc9316: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-fc9316\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-fc9316 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:13:02Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"namespace":"ai-tenant-e2e-worker-w0-fc9316","name":"default-tenant","reconcileID":"0449efcb-18c4-400b-a0a5-f639f07e7123","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w0-fc9316/payload-processing-e2e-worker-w0-fc9316: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-fc9316\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-fc9316 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:13:03Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w0-fc9316/payload-processing-e2e-worker-w0-fc9316: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-fc9316\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-fc9316 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:13:03Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"namespace":"ai-tenant-e2e-worker-w0-fc9316","name":"default-tenant","reconcileID":"0b959f58-aa6a-4a2d-ae09-4bc75051f06a","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w0-fc9316/payload-processing-e2e-worker-w0-fc9316: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-fc9316\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-fc9316 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:13:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"error","ts":"2026-11-03T01:13:04Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w0-fc9316/payload-processing-e2e-worker-w0-fc9316: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-fc9316\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-fc9316 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:13:04Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"namespace":"ai-tenant-e2e-worker-w0-fc9316","name":"default-tenant","reconcileID":"59ba8814-2c17-4276-bf8f-5e050ef8defc","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w0-fc9316/payload-processing-e2e-worker-w0-fc9316: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-fc9316\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-fc9316 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:13:04Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"}}
{"level":"info","ts":"2026-11-03T01:13:05Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-fc9316"},"tenantID":"e2e-worker-w0-fc9316","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:13:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"error","ts":"2026-11-03T01:13:06Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0b4a2007"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:13:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:13:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:13:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:13:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:13:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:13:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:13:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:13:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:13:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:13:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:13:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:13:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:13:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:13:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:13:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:13:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:13:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fdacdb"},"tenantID":"e2e-extmod-b-fdacdb","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-fdacdb"}
{"level":"info","ts":"2026-11-03T01:13:33Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fdacdb"},"tenantID":"e2e-extmod-b-fdacdb","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:13:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:13:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:13:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:13:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:13:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:13:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:13:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:13:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:13:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:13:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:13:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:13:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:13:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:13:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:13:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:14:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:14:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:14:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:14:01Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:14:02Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"tenantID":"e2e-shared-a-w2-065de0","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:14:02Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-065de0"},"namespace":"ai-tenant-e2e-shared-a-w2-065de0","name":"default-tenant","reconcileID":"9902925c-3a7d-412c-b7a5-f95f35d15009","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:14:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:14:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:14:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:14:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:14:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:14:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:14:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:14:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:14:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:14:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:14:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:14:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:14:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:14:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:14:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:14:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:14:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:14:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:14:31Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-7ef1b6"},"tenantID":"e2e-shared-b-w2-7ef1b6","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:14:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:14:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:14:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:14:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:14:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:14:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:14:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:14:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:14:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:14:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:14:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:14:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:14:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"}}
{"level":"info","ts":"2026-11-03T01:14:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:14:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:15:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:15:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:15:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"error","ts":"2026-11-03T01:15:03Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-2f5939b0"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:15:06Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-4c61c8"}}
{"level":"info","ts":"2026-11-03T01:15:07Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-4c61c8"},"tenantID":"e2e-worker-w2-4c61c8","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:15:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:15:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:15:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:15:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:15:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:15:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:15:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-9908e8"}
{"level":"info","ts":"2026-11-03T01:15:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:15:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-20109d"}
{"level":"info","ts":"2026-11-03T01:15:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:15:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c738e0"},"tenantID":"e2e-extmod-a-c738e0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-c738e0"}
{"level":"error","ts":"2026-11-03T01:15:19Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-850433"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-850433/payload-processing-e2e-extmod-a-850433: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-850433\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-850433 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:15:19Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-850433"},"namespace":"ai-tenant-e2e-extmod-a-850433","name":"default-tenant","reconcileID":"11f3b21e-22b5-44e1-bf4d-57bd9bcd1cb0","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-850433/payload-processing-e2e-extmod-a-850433: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-850433\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-850433 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:15:20Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-850433"},"tenantID":"e2e-extmod-a-850433","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:15:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:15:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:15:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:15:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:15:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:15:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:15:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:15:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:15:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:15:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:15:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:15:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:15:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:15:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:15:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:15:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:15:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"tenantID":"e2e-shared-b-w1-5c109b","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-5c109b"}
{"level":"info","ts":"2026-11-03T01:15:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"tenantID":"e2e-shared-b-w1-5c109b","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-5c109b"}
{"level":"info","ts":"2026-11-03T01:15:47Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-ac338d"},"tenantID":"e2e-extmod-b-ac338d","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:15:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-0db47e"},"tenantID":"e2e-extmod-b-0db47e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-0db47e"}
{"level":"info","ts":"2026-11-03T01:15:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:15:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c738e0"},"tenantID":"e2e-extmod-a-c738e0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-c738e0"}
{"level":"info","ts":"2026-11-03T01:15:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c738e0"},"tenantID":"e2e-extmod-a-c738e0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-c738e0"}
{"level":"info","ts":"2026-11-03T01:15:50Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c738e0"},"tenantID":"e2e-extmod-a-c738e0","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:15:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:15:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:15:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:15:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:15:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:16:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:16:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:16:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:16:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:16:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:16:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-0db47e"},"tenantID":"e2e-extmod-b-0db47e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-0db47e"}
{"level":"info","ts":"2026-11-03T01:16:10Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-0db47e"},"tenantID":"e2e-extmod-b-0db47e","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:16:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:16:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:16:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:16:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:16:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:16:19Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:16:19Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"tenantID":"e2e-shared-a-w4-9908e8","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:16:19Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-9908e8"},"namespace":"ai-tenant-e2e-shared-a-w4-9908e8","name":"default-tenant","reconcileID":"f407890e-41e4-4a6e-8c6a-7e56477e32ab","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:16:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:16:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:16:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:16:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:16:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:16:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-099583"}
{"level":"info","ts":"2026-11-03T01:16:30Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"}}
{"level":"info","ts":"2026-11-03T01:16:31Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-099583"},"tenantID":"e2e-shared-a-w1-099583","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:16:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:16:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"info","ts":"2026-11-03T01:16:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"}}
{"level":"info","ts":"2026-11-03T01:16:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:16:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"info","ts":"2026-11-03T01:16:38Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-20109d"},"tenantID":"e2e-shared-b-w4-20109d","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:16:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:16:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"}}
{"level":"error","ts":"2026-11-03T01:16:41Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4ff65120"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:16:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:16:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-99bdb8"},"tenantID":"e2e-worker-w4-99bdb8","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-99bdb8"}
{"level":"info","ts":"2026-11-03T01:16:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"}}
{"level":"error","ts":"2026-11-03T01:16:51Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w1-5c109b/payload-processing-e2e-shared-b-w1-5c109b: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w1-5c109b\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w1-5c109b because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:16:51Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"namespace":"ai-tenant-e2e-shared-b-w1-5c109b","name":"default-tenant","reconcileID":"5b9c269d-5037-4461-92f0-d764d1114149","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w1-5c109b/payload-processing-e2e-shared-b-w1-5c109b: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w1-5c109b\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w1-5c109b because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:16:52Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-5c109b"},"tenantID":"e2e-shared-b-w1-5c109b","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:16:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"error","ts":"2026-11-03T01:16:52Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-121ed61d"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:16:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"error","ts":"2026-11-03T01:16:58Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-1589f2ac"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:16:59Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-99bdb8"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w4-99bdb8/payload-processing-e2e-worker-w4-99bdb8: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-99bdb8\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-99bdb8 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T01:16:59Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-99bdb8"},"namespace":"ai-tenant-e2e-worker-w4-99bdb8","name":"default-tenant","reconcileID":"5a7c6add-13a1-4a35-810d-6b3bd81e79c6","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w4-99bdb8/payload-processing-e2e-worker-w4-99bdb8: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-99bdb8\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-99bdb8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:17:00Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-99bdb8"},"tenantID":"e2e-worker-w4-99bdb8","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:17:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:17:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:17:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-b85343"},"tenantID":"e2e-worker-w1-b85343","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-b85343"}
{"level":"info","ts":"2026-11-03T01:17:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-b85343"},"tenantID":"e2e-worker-w1-b85343","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T01:17:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-b85343"},"tenantID":"e2e-worker-w1-b85343","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T01:17:11Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-b85343"},"namespace":"ai-tenant-e2e-worker-w1-b85343","name":"default-tenant","reconcileID":"92fdd6d9-d524-4a44-9713-4ce670fdae3d","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T01:17:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:17:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:17:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T01:17:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:17:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:17:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:17:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}
{"level":"info","ts":"2026-11-03T01:17:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-540797c3"}}
{"level":"info","ts":"2026-11-03T01:17:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-c6e808d7"}}