{"level":"info","ts":"2026-11-01T12:55:17Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-f6b043"},"tenantID":"e2e-shared-a-w2-f6b043","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:55:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-5b1bd4"},"tenantID":"e2e-worker-w1-5b1bd4","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-5b1bd4"} {"level":"info","ts":"2026-11-01T12:55:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-6b9030"},"tenantID":"e2e-api-a-6b9030","namespace":"openshift-ingress","gatewayName":"e2e-api-a-6b9030"} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d01bdd95"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-58b140b5"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:55:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:55:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-6b9030"},"tenantID":"e2e-api-a-6b9030","namespace":"openshift-ingress","gatewayName":"e2e-api-a-6b9030"} {"level":"info","ts":"2026-11-01T12:55:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w5-9514b0"},"tenantID":"e2e-worker-w5-9514b0","namespace":"openshift-ingress","gatewayName":"e2e-worker-w5-9514b0"} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d01bdd95"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-58b140b5"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:55:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:55:43Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T12:55:44Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d01bdd95"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-58b140b5"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:55:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:55:54Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T12:55:57Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-f893c2"}} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"error","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-d01bdd95"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-58b140b5"}} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:55:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:56:03Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-f893c2"}} {"level":"info","ts":"2026-11-01T12:56:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0a0610"},"tenantID":"e2e-inf-a-0a0610","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-0a0610"} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-f893c2"},"tenantID":"e2e-api-a-f893c2","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-58b140b5"}} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:56:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:56:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0a0610"},"tenantID":"e2e-inf-a-0a0610","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-0a0610"} {"level":"info","ts":"2026-11-01T12:56:11Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-c4bbec"}} {"level":"info","ts":"2026-11-01T12:56:12Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-c4bbec"}} {"level":"info","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"error","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-58b140b5"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:56:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:56:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"},"tenantID":"e2e-shared-b-w4-78c69d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-78c69d"} {"level":"info","ts":"2026-11-01T12:56:25Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-c4bbec"},"tenantID":"e2e-shared-b-w2-c4bbec","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"},"tenantID":"e2e-shared-b-w4-78c69d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-78c69d"} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:56:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:56:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:56:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:56:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:56:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:56:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:56:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:56:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"},"tenantID":"e2e-api-b-2810c2","namespace":"openshift-ingress","gatewayName":"e2e-api-b-2810c2"} {"level":"info","ts":"2026-11-01T12:56:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w6-f7427d"},"tenantID":"e2e-worker-w6-f7427d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w6-f7427d"} {"level":"info","ts":"2026-11-01T12:56:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"},"tenantID":"e2e-api-b-2810c2","namespace":"openshift-ingress","gatewayName":"e2e-api-b-2810c2"} {"level":"info","ts":"2026-11-01T12:56:48Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-c7206f"}} {"level":"info","ts":"2026-11-01T12:56:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:56:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:56:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:56:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:56:49Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-c7206f"}} {"level":"info","ts":"2026-11-01T12:56:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:56:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:56:53Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-6b9030"}} {"level":"info","ts":"2026-11-01T12:56:55Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-6b9030"}} {"level":"info","ts":"2026-11-01T12:56:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:56:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:56:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:56:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:57:00Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-c7206f"}} {"level":"info","ts":"2026-11-01T12:57:02Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-c7206f"},"tenantID":"e2e-worker-w2-c7206f","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:57:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:57:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:57:05Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-6b9030"}} {"level":"info","ts":"2026-11-01T12:57:08Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-6b9030"},"tenantID":"e2e-api-a-6b9030","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:57:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:57:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:57:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:57:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:57:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:57:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:57:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:57:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:57:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"}} {"level":"info","ts":"2026-11-01T12:57:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:57:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:57:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:57:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T12:57:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0a0610"}} {"level":"info","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"error","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-0bd31f5b"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:57:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:57:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0cd09f"},"tenantID":"e2e-shared-b-w1-0cd09f","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-0cd09f"} {"level":"info","ts":"2026-11-01T12:57:42Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0a0610"}} {"level":"info","ts":"2026-11-01T12:57:45Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-0a0610"},"tenantID":"e2e-inf-a-0a0610","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:57:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:57:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:57:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:57:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:57:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:57:47Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-6c650b"}} {"level":"info","ts":"2026-11-01T12:57:47Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-6124f9"}} {"level":"info","ts":"2026-11-01T12:57:48Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-6c650b"}} {"level":"info","ts":"2026-11-01T12:57:49Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-6124f9"}} {"level":"info","ts":"2026-11-01T12:57:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"},"tenantID":"e2e-shared-b-w4-78c69d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-78c69d"} {"level":"info","ts":"2026-11-01T12:57:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"},"tenantID":"e2e-shared-b-w4-78c69d","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-78c69d"} {"level":"info","ts":"2026-11-01T12:58:01Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-6c650b"},"tenantID":"e2e-shared-a-w1-6c650b","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:58:04Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-6124f9"},"tenantID":"e2e-shared-a-w4-6124f9","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:58:05Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-94c13a"}} {"level":"info","ts":"2026-11-01T12:58:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:58:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:58:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:58:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:58:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:58:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"},"tenantID":"e2e-api-b-2810c2","namespace":"openshift-ingress","gatewayName":"e2e-api-b-2810c2"} {"level":"info","ts":"2026-11-01T12:58:16Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-94c13a"},"tenantID":"e2e-api-a-94c13a","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:58:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"},"tenantID":"e2e-api-b-2810c2","namespace":"openshift-ingress","gatewayName":"e2e-api-b-2810c2"} {"level":"info","ts":"2026-11-01T12:58:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:58:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:58:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:58:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"}} {"level":"info","ts":"2026-11-01T12:58:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:58:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:58:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:58:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"error","ts":"2026-11-01T12:58:29Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3a1b6e60"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T12:58:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:58:33Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0cd09f"}} {"level":"info","ts":"2026-11-01T12:58:34Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0cd09f"}} {"level":"info","ts":"2026-11-01T12:58:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:58:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:58:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"}} {"level":"info","ts":"2026-11-01T12:58:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:58:45Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0cd09f"}} {"level":"info","ts":"2026-11-01T12:58:47Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-0cd09f"},"tenantID":"e2e-shared-b-w1-0cd09f","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:58:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T12:58:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:58:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"error","ts":"2026-11-01T12:58:50Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-f466e040"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T12:58:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:58:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T12:59:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:59:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:59:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-94c94b"}} {"level":"info","ts":"2026-11-01T12:59:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"}} {"level":"info","ts":"2026-11-01T12:59:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress","gatewayName":"e2e-mt-187d06ed"} {"level":"info","ts":"2026-11-01T12:59:14Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-94c94b"}} {"level":"info","ts":"2026-11-01T12:59:16Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"}} {"level":"info","ts":"2026-11-01T12:59:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress","gatewayName":"e2e-mt-187d06ed"} {"level":"info","ts":"2026-11-01T12:59:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:59:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:59:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:59:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-94c94b"}} {"level":"info","ts":"2026-11-01T12:59:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-78c69d"},"tenantID":"e2e-shared-b-w4-78c69d","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:59:28Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-94c94b"},"tenantID":"e2e-worker-w0-94c94b","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T12:59:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:59:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:59:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:59:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:59:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:59:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:59:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T12:59:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T12:59:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T12:59:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T12:59:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T12:59:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T12:59:56Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-5b1bd4"}} {"level":"info","ts":"2026-11-01T12:59:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress","gatewayName":"e2e-mt-187d06ed"} {"level":"info","ts":"2026-11-01T13:00:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:00:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T13:00:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:00:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:00:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress","gatewayName":"e2e-mt-187d06ed"} {"level":"info","ts":"2026-11-01T13:00:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:00:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-ecd640"},"tenantID":"e2e-worker-w3-ecd640","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-ecd640"} {"level":"info","ts":"2026-11-01T13:00:13Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-5b1bd4"},"tenantID":"e2e-worker-w1-5b1bd4","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:00:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:00:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"}} {"level":"info","ts":"2026-11-01T13:00:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:00:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-06fd6b"},"tenantID":"e2e-worker-w4-06fd6b","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-06fd6b"} {"level":"info","ts":"2026-11-01T13:00:16Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"}} {"level":"info","ts":"2026-11-01T13:00:17Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:17Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:00:17Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:17Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:00:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"error","ts":"2026-11-01T13:00:23Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-36933efd"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T13:00:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:00:27Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"}} {"level":"info","ts":"2026-11-01T13:00:29Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-2810c2"},"tenantID":"e2e-api-b-2810c2","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:00:29Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:29Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:00:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:00:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:00:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w5-9514b0"},"tenantID":"e2e-worker-w5-9514b0","namespace":"openshift-ingress","gatewayName":"e2e-worker-w5-9514b0"} {"level":"info","ts":"2026-11-01T13:00:37Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:37Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:37Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:39Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:00:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:00:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:00:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:00:49Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:00:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress","gatewayName":"e2e-mt-187d06ed"} {"level":"info","ts":"2026-11-01T13:00:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:00:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress","gatewayName":"e2e-mt-187d06ed"} {"level":"info","ts":"2026-11-01T13:00:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:00:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:01:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:01:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:01:00Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:01:02Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T13:01:03Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T13:01:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:01:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:01:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:01:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:01:14Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"}} {"level":"info","ts":"2026-11-01T13:01:15Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w5-9514b0"}} {"level":"info","ts":"2026-11-01T13:01:18Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-187d06ed"},"tenantID":"e2e-mt-187d06ed","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:01:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:01:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:01:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:01:20Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:01:25Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w5-9514b0"}} {"level":"info","ts":"2026-11-01T13:01:27Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w5-9514b0"},"tenantID":"e2e-worker-w5-9514b0","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:01:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:01:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"}} {"level":"info","ts":"2026-11-01T13:01:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:01:30Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:01:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:01:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:01:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"error","ts":"2026-11-01T13:01:41Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-47feb945"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T13:01:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:01:41Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:01:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T13:01:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w6-f7427d"},"tenantID":"e2e-worker-w6-f7427d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w6-f7427d"} {"level":"info","ts":"2026-11-01T13:01:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T13:01:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:01:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:01:51Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:02:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:02:01Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"}} {"level":"info","ts":"2026-11-01T13:02:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:02:11Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:19Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:19Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:19Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"error","ts":"2026-11-01T13:02:21Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-24fc6e85"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T13:02:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:02:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:02:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:02:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:02:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:02:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:02:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:02:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:03:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:03:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:03:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-ecd640"}} {"level":"info","ts":"2026-11-01T13:03:25Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-ecd640"}} {"level":"info","ts":"2026-11-01T13:03:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:03:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:36Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-ecd640"}} {"level":"info","ts":"2026-11-01T13:03:38Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-ecd640"},"tenantID":"e2e-worker-w3-ecd640","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:03:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:03:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:44Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-06fd6b"}} {"level":"info","ts":"2026-11-01T13:03:45Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-06fd6b"}} {"level":"info","ts":"2026-11-01T13:03:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:03:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:03:56Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-06fd6b"}} {"level":"info","ts":"2026-11-01T13:03:58Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-06fd6b"},"tenantID":"e2e-worker-w4-06fd6b","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:04:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:04:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:04:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:04:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:04:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:04:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:04:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:04:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:04:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:04:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:04:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:04:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:04:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:05:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:05:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:05:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T13:05:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-72078a"} {"level":"info","ts":"2026-11-01T13:05:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:05:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:05:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:05:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:05:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:05:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:05:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:05:40Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"}} {"level":"info","ts":"2026-11-01T13:05:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-b68322"} {"level":"info","ts":"2026-11-01T13:05:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:05:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:05:49Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"}} {"level":"info","ts":"2026-11-01T13:05:51Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-72078a"},"tenantID":"e2e-inf-a-72078a","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:05:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:05:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:06:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:06:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:06:06Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"}} {"level":"info","ts":"2026-11-01T13:06:07Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"}} {"level":"info","ts":"2026-11-01T13:06:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:06:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:06:18Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"}} {"level":"info","ts":"2026-11-01T13:06:20Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-b68322"},"tenantID":"e2e-inf-b-b68322","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:06:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:06:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:06:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:06:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:06:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:06:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:06:40Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w6-f7427d"}} {"level":"info","ts":"2026-11-01T13:06:41Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w6-f7427d"}} {"level":"info","ts":"2026-11-01T13:06:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:06:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:06:52Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w6-f7427d"}} {"level":"info","ts":"2026-11-01T13:06:54Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w6-f7427d"},"tenantID":"e2e-worker-w6-f7427d","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:06:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:06:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:07:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:07:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:07:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:07:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:07:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:07:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:07:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:07:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:07:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:07:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:07:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:07:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:08:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:08:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:08:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:08:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:08:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:08:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:08:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:08:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:08:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:08:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:08:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:08:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:09:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:09:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:09:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:09:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:09:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:09:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:09:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:09:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:09:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:09:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:09:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:09:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:10:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:10:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:10:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:10:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:10:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:10:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:10:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:10:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"}} {"level":"info","ts":"2026-11-01T13:10:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"error","ts":"2026-11-01T13:10:44Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-338d43d4"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T13:10:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:11:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:11:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:11:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:11:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:11:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:11:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:11:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:12:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"info","ts":"2026-11-01T13:12:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"}} {"level":"error","ts":"2026-11-01T13:12:24Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4501696d"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T13:13:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:13:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:14:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:14:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:14:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:14:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:15:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:15:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-483de8"} {"level":"info","ts":"2026-11-01T13:16:23Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"}} {"level":"info","ts":"2026-11-01T13:16:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"}} {"level":"info","ts":"2026-11-01T13:16:34Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"}} {"level":"info","ts":"2026-11-01T13:16:36Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress"} {"level":"error","ts":"2026-11-01T13:16:36Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"namespace":"ai-tenant-e2e-worker-single-main-483de8","name":"default-tenant","reconcileID":"d367a733-83c1-4bac-acc4-b33fb44710fd","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"} {"level":"info","ts":"2026-11-01T13:16:38Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-483de8"},"tenantID":"e2e-worker-single-main-483de8","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:16:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:16:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:17:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:19:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:19:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:19:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:19:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:19:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:20:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:20:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:20:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-f38349"} {"level":"info","ts":"2026-11-01T13:20:56Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"}} {"level":"info","ts":"2026-11-01T13:20:57Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"}} {"level":"info","ts":"2026-11-01T13:21:07Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"}} {"level":"info","ts":"2026-11-01T13:21:10Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-f38349"},"tenantID":"e2e-worker-single-main-f38349","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:22:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:23:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:23:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:23:55Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}} {"level":"info","ts":"2026-11-01T13:23:56Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}} {"level":"info","ts":"2026-11-01T13:23:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:23:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:24:00Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}} {"level":"info","ts":"2026-11-01T13:24:01Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}} {"level":"info","ts":"2026-11-01T13:24:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:24:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:24:10Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}} {"level":"info","ts":"2026-11-01T13:24:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"},"tenantID":"e2e-ait-f3c81bef","namespace":"openshift-ingress","gatewayName":"e2e-ait-f3c81bef"} {"level":"info","ts":"2026-11-01T13:24:10Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}} {"level":"info","ts":"2026-11-01T13:24:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:24:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"},"tenantID":"e2e-ait-f3c81bef","namespace":"openshift-ingress","gatewayName":"e2e-ait-f3c81bef"} {"level":"info","ts":"2026-11-01T13:24:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"} {"level":"info","ts":"2026-11-01T13:24:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"},"tenantID":"e2e-ait-f3c81bef","namespace":"openshift-ingress","gatewayName":"e2e-ait-f3c81bef"} {"level":"info","ts":"2026-11-01T13:24:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"},"tenantID":"e2e-ait-f3c81bef","namespace":"openshift-ingress","gatewayName":"e2e-ait-f3c81bef"} {"level":"info","ts":"2026-11-01T13:24:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"},"tenantID":"e2e-ait-f3c81bef","namespace":"openshift-ingress","gatewayName":"e2e-ait-f3c81bef"} {"level":"info","ts":"2026-11-01T13:24:29Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"}} {"level":"info","ts":"2026-11-01T13:24:40Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"}} {"level":"info","ts":"2026-11-01T13:24:49Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"}} {"level":"info","ts":"2026-11-01T13:24:51Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-f3c81bef"},"tenantID":"e2e-ait-f3c81bef","namespace":"openshift-ingress"} {"level":"info","ts":"2026-11-01T13:25:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress","gatewayName":"e2e-authcfg-http-624dd4"} {"level":"info","ts":"2026-11-01T13:25:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress","gatewayName":"e2e-authcfg-http-624dd4"} {"level":"info","ts":"2026-11-01T13:25:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress","gatewayName":"e2e-authcfg-http-624dd4"} {"level":"info","ts":"2026-11-01T13:25:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress","gatewayName":"e2e-authcfg-http-624dd4"} {"level":"info","ts":"2026-11-01T13:25:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress","gatewayName":"e2e-authcfg-http-624dd4"} {"level":"info","ts":"2026-11-01T13:25:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress","gatewayName":"e2e-authcfg-http-624dd4"} {"level":"info","ts":"2026-11-01T13:26:52Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"}} {"level":"info","ts":"2026-11-01T13:26:53Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"}} {"level":"info","ts":"2026-11-01T13:27:03Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"}} {"level":"info","ts":"2026-11-01T13:27:05Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-authcfg-http-624dd4"},"tenantID":"e2e-authcfg-http-624dd4","namespace":"openshift-ingress"}