{"level":"info","ts":"2026-11-03T21:22:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:22:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"error","ts":"2026-11-03T21:22:47Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-delete-dded24","namespace":"e2e-extmod-att-user-7db70b"},"namespace":"e2e-extmod-att-user-7db70b","name":"e2e-att-delete-dded24","reconcileID":"7a716d9a-70ce-4029-8273-02b602d76f22","error":"update ExternalModel attachment finalizer: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-att-delete-dded24\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:22:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:22:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:22:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:22:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:22:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:22:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-6ac027"},"tenantID":"e2e-inf-b-6ac027","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-6ac027"}
{"level":"info","ts":"2026-11-03T21:22:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-6ac027"},"tenantID":"e2e-inf-b-6ac027","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-6ac027"}
{"level":"info","ts":"2026-11-03T21:22:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:02Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-68c77a"},"tenantID":"e2e-inf-a-68c77a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:23:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"error","ts":"2026-11-03T21:23:15Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-6ac027"},"error":"apply NetworkPolicy ai-tenant-e2e-inf-b-6ac027/payload-processing-e2e-inf-b-6ac027: networkpolicies.networking.k8s.io \"payload-processing-e2e-inf-b-6ac027\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-6ac027 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:23:15Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-6ac027"},"namespace":"ai-tenant-e2e-inf-b-6ac027","name":"default-tenant","reconcileID":"05864aa2-f0b1-40a1-9421-a77df06f91d5","error":"apply: apply NetworkPolicy ai-tenant-e2e-inf-b-6ac027/payload-processing-e2e-inf-b-6ac027: networkpolicies.networking.k8s.io \"payload-processing-e2e-inf-b-6ac027\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-6ac027 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:23:16Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-6ac027"},"tenantID":"e2e-inf-b-6ac027","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:23:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:23:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress","gatewayName":"e2e-ait-03b486b1"}
{"level":"info","ts":"2026-11-03T21:23:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress","gatewayName":"e2e-ait-03b486b1"}
{"level":"info","ts":"2026-11-03T21:23:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress","gatewayName":"e2e-ait-03b486b1"}
{"level":"info","ts":"2026-11-03T21:23:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress","gatewayName":"e2e-ait-03b486b1"}
{"level":"info","ts":"2026-11-03T21:24:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress","gatewayName":"e2e-ait-03b486b1"}
{"level":"info","ts":"2026-11-03T21:24:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress","gatewayName":"e2e-ait-03b486b1"}
{"level":"error","ts":"2026-11-03T21:24:14Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rec-recover-639bf3","namespace":"e2e-extmod-rec-noaccess-c611fa"},"namespace":"e2e-extmod-rec-noaccess-c611fa","name":"e2e-rec-recover-639bf3","reconcileID":"298bd5bc-c3d4-4601-a720-2d0bcb8d1dc9","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-rec-recover-639bf3\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:24:18Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"}}
{"level":"info","ts":"2026-11-03T21:24:19Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"}}
{"level":"error","ts":"2026-11-03T21:24:26Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rec-ctrl-78fda8","namespace":"e2e-extmod-rec-user-9c9c54"},"namespace":"e2e-extmod-rec-user-9c9c54","name":"e2e-rec-ctrl-78fda8","reconcileID":"c90dcf35-3355-4dec-bfcf-36834ea7bb8c","error":"update ExternalModel attachment finalizer: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-rec-ctrl-78fda8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:24:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:24:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:24:30Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-03b486b1"},"tenantID":"e2e-ait-03b486b1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:24:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:24:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:24:50Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"}}
{"level":"info","ts":"2026-11-03T21:24:50Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"}}
{"level":"info","ts":"2026-11-03T21:24:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress","gatewayName":"e2e-migrate-03237dd6-gw"}
{"level":"info","ts":"2026-11-03T21:24:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress","gatewayName":"e2e-migrate-03237dd6-gw"}
{"level":"info","ts":"2026-11-03T21:24:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:24:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:25:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:25:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:25:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:25:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:25:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress","gatewayName":"e2e-migrate-03237dd6-gw"}
{"level":"info","ts":"2026-11-03T21:25:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress","gatewayName":"e2e-migrate-03237dd6-gw"}
{"level":"info","ts":"2026-11-03T21:25:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:25:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:25:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:25:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:25:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress","gatewayName":"e2e-migrate-03237dd6-gw"}
{"level":"info","ts":"2026-11-03T21:25:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress","gatewayName":"e2e-migrate-03237dd6-gw"}
{"level":"info","ts":"2026-11-03T21:25:25Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"}}
{"level":"info","ts":"2026-11-03T21:25:25Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"}}
{"level":"info","ts":"2026-11-03T21:25:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:25:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:25:36Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-03237dd6"},"tenantID":"e2e-migrate-03237dd6","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:25:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress","gatewayName":"e2e-ait-4e4dcdb1"}
{"level":"info","ts":"2026-11-03T21:25:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress","gatewayName":"e2e-ait-4e4dcdb1"}
{"level":"info","ts":"2026-11-03T21:25:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:25:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:25:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:25:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress","gatewayName":"e2e-ait-4e4dcdb1"}
{"level":"info","ts":"2026-11-03T21:25:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress","gatewayName":"e2e-ait-4e4dcdb1"}
{"level":"info","ts":"2026-11-03T21:26:02Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"}}
{"level":"info","ts":"2026-11-03T21:26:02Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"}}
{"level":"info","ts":"2026-11-03T21:26:02Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"}}
{"level":"error","ts":"2026-11-03T21:26:03Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rot-0d60b4","namespace":"e2e-extmod-rot-user-1dd4f7"},"namespace":"e2e-extmod-rot-user-1dd4f7","name":"e2e-rot-0d60b4","reconcileID":"a443d5dd-16ab-4e9e-bc26-1429bc8bc1bc","error":"apply Service e2e-extmod-rot-user-1dd4f7/provider-e2e-rot-0d60b4-provider: services \"provider-e2e-rot-0d60b4-provider\" is forbidden: unable to create new content in namespace e2e-extmod-rot-user-1dd4f7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:26:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:26:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:26:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:26:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:26:12Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"}}
{"level":"info","ts":"2026-11-03T21:26:13Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:26:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress","gatewayName":"e2e-ait-4e4dcdb1"}
{"level":"info","ts":"2026-11-03T21:26:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress","gatewayName":"e2e-ait-4e4dcdb1"}
{"level":"info","ts":"2026-11-03T21:26:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:26:20Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"}}
{"level":"info","ts":"2026-11-03T21:26:20Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"}}
{"level":"info","ts":"2026-11-03T21:26:30Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"}}
{"level":"info","ts":"2026-11-03T21:26:30Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-4e4dcdb1"},"tenantID":"e2e-ait-4e4dcdb1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:26:42Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"}}
{"level":"info","ts":"2026-11-03T21:26:42Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"}}
{"level":"info","ts":"2026-11-03T21:26:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:26:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"error","ts":"2026-11-03T21:26:46Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-noaccess-626bce","namespace":"e2e-extmod-att-noaccess-26883d"},"namespace":"e2e-extmod-att-noaccess-26883d","name":"e2e-att-noaccess-626bce","reconcileID":"f9b6a1e6-7493-45e6-a62a-b29ecc72c620","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-att-noaccess-626bce\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:26:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:26:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:26:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"},"tenantID":"e2e-ait-db037162","namespace":"openshift-ingress","gatewayName":"e2e-ait-db037162"}
{"level":"info","ts":"2026-11-03T21:26:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"},"tenantID":"e2e-ait-db037162","namespace":"openshift-ingress","gatewayName":"e2e-ait-db037162"}
{"level":"info","ts":"2026-11-03T21:26:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:27:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"},"tenantID":"e2e-ait-db037162","namespace":"openshift-ingress","gatewayName":"e2e-ait-db037162"}
{"level":"info","ts":"2026-11-03T21:27:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress","gatewayName":"e2e-mt-abdb6c3e"}
{"level":"info","ts":"2026-11-03T21:27:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"},"tenantID":"e2e-ait-db037162","namespace":"openshift-ingress","gatewayName":"e2e-ait-db037162"}
{"level":"error","ts":"2026-11-03T21:27:11Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-unrel-bb5286","namespace":"e2e-extmod-att-user-7db70b"},"namespace":"e2e-extmod-att-user-7db70b","name":"e2e-att-unrel-bb5286","reconcileID":"58845d9b-f455-414a-b64c-a7de02220227","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-att-unrel-bb5286\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:27:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:27:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:27:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-ff49f4"},"tenantID":"e2e-worker-w0-ff49f4","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-ff49f4"}
{"level":"error","ts":"2026-11-03T21:27:22Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-serve-aa8a44","namespace":"e2e-extmod-att-user-7db70b"},"namespace":"e2e-extmod-att-user-7db70b","name":"e2e-att-serve-aa8a44","reconcileID":"8397e368-d9f0-4609-9016-45dfc53c2588","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-att-serve-aa8a44\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:27:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:27:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"},"tenantID":"e2e-ait-db037162","namespace":"openshift-ingress","gatewayName":"e2e-ait-db037162"}
{"level":"info","ts":"2026-11-03T21:27:27Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"}}
{"level":"info","ts":"2026-11-03T21:27:28Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"}}
{"level":"info","ts":"2026-11-03T21:27:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:27:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:27:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:27:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:27:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-6f34cf"},"tenantID":"e2e-worker-w3-6f34cf","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-6f34cf"}
{"level":"info","ts":"2026-11-03T21:27:39Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-db037162"},"tenantID":"e2e-ait-db037162","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:27:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:27:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress","gatewayName":"e2e-derive-333b6c01"}
{"level":"info","ts":"2026-11-03T21:27:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress","gatewayName":"e2e-derive-333b6c01"}
{"level":"info","ts":"2026-11-03T21:27:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:27:54Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-abdb6c3e"},"tenantID":"e2e-mt-abdb6c3e","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:27:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:27:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:28:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress","gatewayName":"e2e-derive-333b6c01"}
{"level":"info","ts":"2026-11-03T21:28:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress","gatewayName":"e2e-derive-333b6c01"}
{"level":"info","ts":"2026-11-03T21:28:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:28:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:28:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:28:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress","gatewayName":"e2e-derive-333b6c01"}
{"level":"info","ts":"2026-11-03T21:28:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress","gatewayName":"e2e-derive-333b6c01"}
{"level":"info","ts":"2026-11-03T21:28:21Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:28:21Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:28:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:23Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"}}
{"level":"info","ts":"2026-11-03T21:28:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:23Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:28:23Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:28:31Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:28:32Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"}}
{"level":"info","ts":"2026-11-03T21:28:33Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-333b6c01"},"tenantID":"e2e-derive-333b6c01","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:28:33Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:28:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:41Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:28:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress","gatewayName":"e2e-api-a-8f7720"}
{"level":"info","ts":"2026-11-03T21:28:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:28:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress","gatewayName":"e2e-api-a-8f7720"}
{"level":"info","ts":"2026-11-03T21:28:46Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:28:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:28:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:28:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress","gatewayName":"e2e-api-a-8f7720"}
{"level":"info","ts":"2026-11-03T21:28:57Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:28:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress","gatewayName":"e2e-api-a-8f7720"}
{"level":"info","ts":"2026-11-03T21:29:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress","gatewayName":"e2e-autores-6bda62"}
{"level":"info","ts":"2026-11-03T21:29:01Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:29:01Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"tenantID":"e2e-autores-6bda62","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T21:29:01Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-6bda62"},"namespace":"ai-tenant-e2e-autores-6bda62","name":"default-tenant","reconcileID":"97d30c84-0abe-4f70-b005-55f0299a283e","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:29:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:29:07Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:29:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress","gatewayName":"e2e-api-a-8f7720"}
{"level":"info","ts":"2026-11-03T21:29:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress","gatewayName":"e2e-api-a-8f7720"}
{"level":"info","ts":"2026-11-03T21:29:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:29:17Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:29:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress","gatewayName":"e2e-api-b-3a1d18"}
{"level":"info","ts":"2026-11-03T21:29:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress","gatewayName":"e2e-api-b-3a1d18"}
{"level":"info","ts":"2026-11-03T21:29:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:29:27Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:29:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:29:37Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:29:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress","gatewayName":"e2e-api-b-3a1d18"}
{"level":"info","ts":"2026-11-03T21:29:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress","gatewayName":"e2e-api-b-3a1d18"}
{"level":"info","ts":"2026-11-03T21:29:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:29:47Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:29:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:29:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:29:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:29:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress","gatewayName":"e2e-api-b-3a1d18"}
{"level":"info","ts":"2026-11-03T21:30:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:00Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:30:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"info","ts":"2026-11-03T21:30:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress","gatewayName":"e2e-explicit-176672"}
{"level":"error","ts":"2026-11-03T21:30:13Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"error":"apply NetworkPolicy ai-tenant-e2e-api-a-8f7720/payload-processing-e2e-api-a-8f7720: networkpolicies.networking.k8s.io \"payload-processing-e2e-api-a-8f7720\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-8f7720 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:30:13Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"namespace":"ai-tenant-e2e-api-a-8f7720","name":"default-tenant","reconcileID":"e5ba2d5c-4d70-436c-9731-2e24af8598ad","error":"apply: apply NetworkPolicy ai-tenant-e2e-api-a-8f7720/payload-processing-e2e-api-a-8f7720: networkpolicies.networking.k8s.io \"payload-processing-e2e-api-a-8f7720\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-8f7720 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:30:13Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"}}
{"level":"info","ts":"2026-11-03T21:30:14Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-8f7720"},"tenantID":"e2e-api-a-8f7720","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:30:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:30:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:30:20Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:30:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress","gatewayName":"e2e-api-b-3a1d18"}
{"level":"info","ts":"2026-11-03T21:30:34Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-3a1d18"},"tenantID":"e2e-api-b-3a1d18","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:30:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:30:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:30:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:30:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:53Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"}}
{"level":"info","ts":"2026-11-03T21:30:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:54Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-176672"},"tenantID":"e2e-explicit-176672","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:30:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-3f0dde"}
{"level":"info","ts":"2026-11-03T21:30:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:30:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:30:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:31:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:31:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-e45ed0"}
{"level":"info","ts":"2026-11-03T21:31:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:31:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:31:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:31:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:31:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:31:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"error","ts":"2026-11-03T21:31:15Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-broken-b8e33f","namespace":"e2e-extmod-cfl-2-427820"},"namespace":"e2e-extmod-cfl-2-427820","name":"e2e-cfl-broken-b8e33f","reconcileID":"3e22a866-0027-4384-a79a-6b452e64a244","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-cfl-broken-b8e33f\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:31:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:31:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:31:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:31:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:31:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:31:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:31:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:31:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:31:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:31:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:31:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:31:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:31:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:31:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:31:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:31:50Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:31:50Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:31:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"error","ts":"2026-11-03T21:31:53Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-same-169ef8","namespace":"e2e-extmod-cfl-2-427820"},"namespace":"e2e-extmod-cfl-2-427820","name":"e2e-cfl-same-169ef8","reconcileID":"5b3675c7-4054-4bef-b6fa-fce11d77a9e9","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-cfl-same-169ef8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:31:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:31:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:31:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:31:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:32:00Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"error","ts":"2026-11-03T21:32:06Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-ef2ac3","namespace":"e2e-extmod-cfl-1-0da87f"},"namespace":"e2e-extmod-cfl-1-0da87f","name":"e2e-cfl-ef2ac3","reconcileID":"69dcd039-7b33-40d7-a377-2a9f9c2c6adb","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-cfl-ef2ac3\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:32:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:32:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:32:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:32:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:32:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-ff49f4"},"tenantID":"e2e-worker-w0-ff49f4","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:32:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:32:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:32:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:32:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:32:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:32:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:32:24Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:32:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:32:27Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:27Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:27Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:32:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:32:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:32:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-6f34cf"},"tenantID":"e2e-worker-w3-6f34cf","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-6f34cf"}
{"level":"info","ts":"2026-11-03T21:32:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:32:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:32:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:32:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:32:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:32:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:33:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:33:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:33:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:33:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:33:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:33:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:33:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:27Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-3f0dde"},"tenantID":"e2e-shared-a-w4-3f0dde","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:33:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:33:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:33:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:33:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:33:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:33:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:33:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:33:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:33:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:33:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:33:42Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"}}
{"level":"info","ts":"2026-11-03T21:33:43Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-e45ed0"},"tenantID":"e2e-shared-b-w4-e45ed0","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:33:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:33:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:33:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:33:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:33:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:34:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:34:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:34:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:34:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:34:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:34:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:34:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"error","ts":"2026-11-03T21:34:13Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-6f34cf"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w3-6f34cf/payload-processing-e2e-worker-w3-6f34cf: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-6f34cf\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-6f34cf because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:34:13Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-6f34cf"},"namespace":"ai-tenant-e2e-worker-w3-6f34cf","name":"default-tenant","reconcileID":"4e489c7c-3648-4783-97de-3b05862946f1","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w3-6f34cf/payload-processing-e2e-worker-w3-6f34cf: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-6f34cf\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-6f34cf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:34:14Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-6f34cf"}}
{"level":"info","ts":"2026-11-03T21:34:14Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-6f34cf"},"tenantID":"e2e-worker-w3-6f34cf","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:34:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:34:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:34:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:34:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:34:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:34:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"error","ts":"2026-11-03T21:34:38Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-fail-83116560","namespace":"e2e-extmod-fail-user-ba5407"},"namespace":"e2e-extmod-fail-user-ba5407","name":"e2e-fail-83116560","reconcileID":"ae1b3b9b-96e8-4395-8120-07ce85c377e4","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-fail-83116560\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:34:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:34:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:34:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:34:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:34:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:34:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:34:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:34:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:34:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:34:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:34:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:35:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:35:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:35:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:35:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:35:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:35:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:35:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:35:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:35:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:35:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:35:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:35:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:35:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:35:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:35:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:35:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:35:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:35:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:35:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:35:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:35:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:35:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:35:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:35:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:35:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:35:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:35:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:35:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:36:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:36:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:36:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:36:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:36:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:36:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:36:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:36:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:36:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:36:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:36:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:36:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:36:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:36:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:36:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:36:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:36:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:36:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:36:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:36:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:37:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:37:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:37:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:37:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:37:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:37:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"error","ts":"2026-11-03T21:37:19Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-quota-0e6617","namespace":"e2e-extmod-shr-user-b733a2"},"namespace":"e2e-extmod-shr-user-b733a2","name":"e2e-shr-quota-0e6617","reconcileID":"5118ee2f-158d-4003-bb45-61818c6466a3","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-quota-0e6617\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:37:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:37:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:37:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:37:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"error","ts":"2026-11-03T21:37:22Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-amb-9e0d75","namespace":"e2e-extmod-shr-user-b733a2"},"namespace":"e2e-extmod-shr-user-b733a2","name":"e2e-shr-amb-9e0d75","reconcileID":"79280674-dafe-4c38-bb0f-af2d6072cf55","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-amb-9e0d75\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:37:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:37:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:37:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:37:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:37:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:37:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:37:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:37:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:37:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:37:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"error","ts":"2026-11-03T21:37:45Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-auth-84d02e","namespace":"e2e-extmod-shr-user-b733a2"},"namespace":"e2e-extmod-shr-user-b733a2","name":"e2e-shr-auth-84d02e","reconcileID":"d76af9ca-ce2f-4e03-95b1-95be24df7cab","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-auth-84d02e\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:37:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:37:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:37:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:37:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:37:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:37:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:37:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:37:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:37:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:37:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:38:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:38:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:38:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"error","ts":"2026-11-03T21:38:09Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-shared-7883ee","namespace":"e2e-extmod-shr-user-b733a2"},"namespace":"e2e-extmod-shr-user-b733a2","name":"e2e-shr-shared-7883ee","reconcileID":"a6532675-b3b4-4e09-98ef-2e25541d32b8","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-shared-7883ee\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:38:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:38:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:38:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-09ab3e"}
{"level":"info","ts":"2026-11-03T21:38:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-a3235a"}
{"level":"info","ts":"2026-11-03T21:38:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:38:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:38:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"error","ts":"2026-11-03T21:38:21Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-09ab3e/payload-processing-e2e-extmod-a-09ab3e: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-09ab3e\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-09ab3e because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:38:21Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"namespace":"ai-tenant-e2e-extmod-a-09ab3e","name":"default-tenant","reconcileID":"5ecdf750-1033-42e3-9557-4e5a7bf6f339","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-09ab3e/payload-processing-e2e-extmod-a-09ab3e: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-09ab3e\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-09ab3e because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:38:22Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-09ab3e"},"tenantID":"e2e-extmod-a-09ab3e","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:38:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:38:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:38:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:38:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:38:36Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-a3235a"},"tenantID":"e2e-extmod-b-a3235a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:38:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:38:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}
{"level":"info","ts":"2026-11-03T21:38:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-086da939"}}
{"level":"info","ts":"2026-11-03T21:38:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-49ec1494"}}
{"level":"info","ts":"2026-11-03T21:38:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-9c5ba6ff"}}