{"level":"info","ts":"2026-11-03T21:38:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:38:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:38:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a7f34a"},"tenantID":"e2e-worker-w0-a7f34a","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a7f34a"}
{"level":"info","ts":"2026-11-03T21:38:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:39:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:39:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:39:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:39:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:39:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:39:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:39:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:39:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:39:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:39:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:39:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:40:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:40:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:40:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:40:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cdf20e"},"tenantID":"e2e-inf-a-cdf20e","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-cdf20e"}
{"level":"info","ts":"2026-11-03T21:40:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cdf20e"},"tenantID":"e2e-inf-a-cdf20e","namespace":"openshift-ingress","gatewayName":"e2e-inf-a-cdf20e"}
{"level":"info","ts":"2026-11-03T21:40:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:40:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:40:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:40:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:40:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:40:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-089f68"},"tenantID":"e2e-inf-b-089f68","namespace":"openshift-ingress","gatewayName":"e2e-inf-b-089f68"}
{"level":"info","ts":"2026-11-03T21:40:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:40:39Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-a-cdf20e"},"tenantID":"e2e-inf-a-cdf20e","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:40:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:40:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:40:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:40:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:41:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:41:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:41:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:41:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:41:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:41:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:41:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:41:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-089f68"},"tenantID":"e2e-inf-b-089f68","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T21:41:29Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-delete-0cf130","namespace":"e2e-extmod-att-user-2e06bd"},"namespace":"e2e-extmod-att-user-2e06bd","name":"e2e-att-delete-0cf130","reconcileID":"c3e9bc4e-8885-4597-a664-0c7d5cff3ccf","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-att-delete-0cf130\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:41:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:41:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:41:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:41:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:41:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:41:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:41:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:42:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:42:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"},"tenantID":"e2e-ait-a04f11df","namespace":"openshift-ingress","gatewayName":"e2e-ait-a04f11df"}
{"level":"info","ts":"2026-11-03T21:42:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:42:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"},"tenantID":"e2e-ait-a04f11df","namespace":"openshift-ingress","gatewayName":"e2e-ait-a04f11df"}
{"level":"info","ts":"2026-11-03T21:42:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:42:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:42:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:42:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"},"tenantID":"e2e-ait-a04f11df","namespace":"openshift-ingress","gatewayName":"e2e-ait-a04f11df"}
{"level":"info","ts":"2026-11-03T21:42:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:42:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"},"tenantID":"e2e-ait-a04f11df","namespace":"openshift-ingress","gatewayName":"e2e-ait-a04f11df"}
{"level":"info","ts":"2026-11-03T21:43:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"},"tenantID":"e2e-ait-a04f11df","namespace":"openshift-ingress","gatewayName":"e2e-ait-a04f11df"}
{"level":"info","ts":"2026-11-03T21:43:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-e92519"}
{"level":"info","ts":"2026-11-03T21:43:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:43:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"}}
{"level":"error","ts":"2026-11-03T21:43:30Z","logger":"external-model","msg":"update provider status","provider":"e2e-rec-recover-cb540c-provider","error":"externalproviders.inference.opendatahub.io \"e2e-rec-recover-cb540c-provider\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/controller.(*Reconciler).updateProviderStatus\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/controller/reconciler.go:2240\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/controller.(*Reconciler).ReconcileProvider\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/controller/reconciler.go:447\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/controller.(*providerReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/controller/reconciler.go:137\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:43:30Z","msg":"Reconciler error","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","ExternalProvider":{"name":"e2e-rec-recover-cb540c-provider","namespace":"e2e-extmod-rec-noaccess-566ba8"},"namespace":"e2e-extmod-rec-noaccess-566ba8","name":"e2e-rec-recover-cb540c-provider","reconcileID":"bc99c8d4-233b-449e-8285-1caf84058ff3","error":"update provider status e2e-extmod-rec-noaccess-566ba8/e2e-rec-recover-cb540c-provider: externalproviders.inference.opendatahub.io \"e2e-rec-recover-cb540c-provider\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:43:31Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rec-recover-cb540c","namespace":"e2e-extmod-rec-noaccess-566ba8"},"namespace":"e2e-extmod-rec-noaccess-566ba8","name":"e2e-rec-recover-cb540c","reconcileID":"b743208d-2884-4d12-a8c0-cc4bf13fb9d6","error":"update ExternalModel attachment finalizer: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-rec-recover-cb540c\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:43:32Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"}}
{"level":"info","ts":"2026-11-03T21:43:35Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-a04f11df"},"tenantID":"e2e-ait-a04f11df","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:43:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"error","ts":"2026-11-03T21:43:43Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rec-ctrl-41e692","namespace":"e2e-extmod-rec-user-a830cb"},"namespace":"e2e-extmod-rec-user-a830cb","name":"e2e-rec-ctrl-41e692","reconcileID":"03deeae4-6ad0-41b0-8ba4-1aa9882ed9d7","error":"apply HTTPRoute e2e-extmod-rec-user-a830cb/external-model-e2e-rec-ctrl-41e692-ef592cd53598: httproutes.gateway.networking.k8s.io \"external-model-e2e-rec-ctrl-41e692-ef592cd53598\" is forbidden: unable to create new content in namespace e2e-extmod-rec-user-a830cb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T21:43:44Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rec-ctrl-41e692","namespace":"e2e-extmod-rec-user-a830cb"},"namespace":"e2e-extmod-rec-user-a830cb","name":"e2e-rec-ctrl-41e692","reconcileID":"d31724c0-27c3-45b9-8963-4ced079961a0","error":"update ExternalModel attachment finalizer: Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-rec-ctrl-41e692\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:43:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:43:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:43:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:43:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:43:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:43:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a7f34a"},"tenantID":"e2e-worker-w0-a7f34a","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a7f34a"}
{"level":"info","ts":"2026-11-03T21:44:00Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"}}
{"level":"info","ts":"2026-11-03T21:44:00Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"}}
{"level":"info","ts":"2026-11-03T21:44:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"},"tenantID":"e2e-migrate-7fc738a9","namespace":"openshift-ingress","gatewayName":"e2e-migrate-7fc738a9-gw"}
{"level":"info","ts":"2026-11-03T21:44:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"},"tenantID":"e2e-migrate-7fc738a9","namespace":"openshift-ingress","gatewayName":"e2e-migrate-7fc738a9-gw"}
{"level":"info","ts":"2026-11-03T21:44:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-e92519"}
{"level":"info","ts":"2026-11-03T21:44:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:44:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-e92519"}
{"level":"info","ts":"2026-11-03T21:44:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:44:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-e92519"}
{"level":"info","ts":"2026-11-03T21:44:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:44:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"},"tenantID":"e2e-migrate-7fc738a9","namespace":"openshift-ingress","gatewayName":"e2e-migrate-7fc738a9-gw"}
{"level":"info","ts":"2026-11-03T21:44:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"},"tenantID":"e2e-migrate-7fc738a9","namespace":"openshift-ingress","gatewayName":"e2e-migrate-7fc738a9-gw"}
{"level":"info","ts":"2026-11-03T21:44:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:44:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:44:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:44:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:45:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:45:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:45:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:45:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"},"tenantID":"e2e-migrate-7fc738a9","namespace":"openshift-ingress","gatewayName":"e2e-migrate-7fc738a9-gw"}
{"level":"info","ts":"2026-11-03T21:45:28Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"}}
{"level":"info","ts":"2026-11-03T21:45:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:45:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:45:41Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-7fc738a9"},"tenantID":"e2e-migrate-7fc738a9","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:45:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:45:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:46:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:46:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:46:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:46:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:46:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:46:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress","gatewayName":"e2e-ait-aff3af59"}
{"level":"info","ts":"2026-11-03T21:46:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress","gatewayName":"e2e-ait-aff3af59"}
{"level":"info","ts":"2026-11-03T21:46:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:46:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:46:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:46:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:46:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:46:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"error","ts":"2026-11-03T21:46:46Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-unrel-12a574","namespace":"e2e-extmod-att-user-2e06bd"},"namespace":"e2e-extmod-att-user-2e06bd","name":"e2e-att-unrel-12a574","reconcileID":"00e7bc58-64d5-4fdc-ba8d-eee3058889f0","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-att-unrel-12a574\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:46:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:46:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:46:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:46:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"error","ts":"2026-11-03T21:46:59Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-att-serve-ce5b2c","namespace":"e2e-extmod-att-user-2e06bd"},"namespace":"e2e-extmod-att-user-2e06bd","name":"e2e-att-serve-ce5b2c","reconcileID":"7dfc611c-d460-43cb-9126-6ea484024e02","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-att-serve-ce5b2c\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:47:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:47:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:47:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress","gatewayName":"e2e-ait-aff3af59"}
{"level":"info","ts":"2026-11-03T21:47:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:47:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress","gatewayName":"e2e-ait-aff3af59"}
{"level":"info","ts":"2026-11-03T21:47:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:47:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:47:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:47:22Z","logger":"tenant","msg":"routing overlay is not ready for ExternalModel ExtProc","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"reason":"routing overlay does not contain provider e2e-cmp-legacy-8f4c38-provider","namespace":"ai-tenant-e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:47:22Z","logger":"tenant","msg":"routing overlay is not ready for ExternalModel ExtProc","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"reason":"routing overlay does not contain provider e2e-cmp-legacy-8f4c38-provider","namespace":"ai-tenant-e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:47:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:47:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:47:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:47:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress","gatewayName":"e2e-autores-4bf46d"}
{"level":"info","ts":"2026-11-03T21:47:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress","gatewayName":"e2e-ait-aff3af59"}
{"level":"info","ts":"2026-11-03T21:47:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress","gatewayName":"e2e-ait-aff3af59"}
{"level":"info","ts":"2026-11-03T21:48:00Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"}}
{"level":"info","ts":"2026-11-03T21:48:01Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"}}
{"level":"info","ts":"2026-11-03T21:48:08Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"}}
{"level":"info","ts":"2026-11-03T21:48:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-autores-4bf46d"},"tenantID":"e2e-autores-4bf46d","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:48:13Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-aff3af59"},"tenantID":"e2e-ait-aff3af59","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:48:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:48:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:48:41Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"}}
{"level":"info","ts":"2026-11-03T21:48:42Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"}}
{"level":"info","ts":"2026-11-03T21:48:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"},"tenantID":"e2e-ait-543258c6","namespace":"openshift-ingress","gatewayName":"e2e-ait-543258c6"}
{"level":"error","ts":"2026-11-03T21:48:52Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cmp-legacy-8f4c38","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"namespace":"ai-tenant-e2e-extmod-a-a4dad5","name":"e2e-cmp-legacy-8f4c38","reconcileID":"1512f708-de77-476c-994d-cb02cca27923","error":"remove ExternalModel cleanup finalizer: externalmodels.inference.opendatahub.io \"e2e-cmp-legacy-8f4c38\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:48:52Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"}}
{"level":"info","ts":"2026-11-03T21:48:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"error","ts":"2026-11-03T21:48:57Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-f83c38","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"namespace":"ai-tenant-e2e-extmod-a-a4dad5","name":"e2e-cfl-f83c38","reconcileID":"20a41f43-f83d-48c0-ad4b-f25699e0f1ce","error":"resolver: no provider refs resolved to routes","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:48:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"},"tenantID":"e2e-ait-543258c6","namespace":"openshift-ingress","gatewayName":"e2e-ait-543258c6"}
{"level":"info","ts":"2026-11-03T21:49:00Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:49:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:49:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a7f34a"},"tenantID":"e2e-worker-w0-a7f34a","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a7f34a"}
{"level":"info","ts":"2026-11-03T21:49:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:49:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:49:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:49:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:49:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"},"tenantID":"e2e-ait-543258c6","namespace":"openshift-ingress","gatewayName":"e2e-ait-543258c6"}
{"level":"info","ts":"2026-11-03T21:49:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"},"tenantID":"e2e-ait-543258c6","namespace":"openshift-ingress","gatewayName":"e2e-ait-543258c6"}
{"level":"info","ts":"2026-11-03T21:49:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-e92519"}
{"level":"info","ts":"2026-11-03T21:49:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:49:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:50:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:50:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-e92519"}
{"level":"info","ts":"2026-11-03T21:50:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:50:07Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"}}
{"level":"info","ts":"2026-11-03T21:50:07Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"}}
{"level":"info","ts":"2026-11-03T21:50:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:50:13Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:50:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:50:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"},"tenantID":"e2e-ait-543258c6","namespace":"openshift-ingress","gatewayName":"e2e-ait-543258c6"}
{"level":"info","ts":"2026-11-03T21:50:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:50:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:50:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"}}
{"level":"info","ts":"2026-11-03T21:50:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:50:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-3880e7"}
{"level":"info","ts":"2026-11-03T21:50:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:50:35Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-543258c6"},"tenantID":"e2e-ait-543258c6","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:50:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:50:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:50:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:50:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:50:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:50:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:50:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:51:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:51:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:51:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:51:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:51:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"},"tenantID":"e2e-derive-8142bf56","namespace":"openshift-ingress","gatewayName":"e2e-derive-8142bf56"}
{"level":"info","ts":"2026-11-03T21:51:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"},"tenantID":"e2e-derive-8142bf56","namespace":"openshift-ingress","gatewayName":"e2e-derive-8142bf56"}
{"level":"info","ts":"2026-11-03T21:51:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:51:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:51:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress","gatewayName":"e2e-explicit-a1c779"}
{"level":"info","ts":"2026-11-03T21:51:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:51:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress","gatewayName":"e2e-mt-fd831e1c"}
{"level":"info","ts":"2026-11-03T21:51:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:51:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:51:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:51:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:51:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"},"tenantID":"e2e-derive-8142bf56","namespace":"openshift-ingress","gatewayName":"e2e-derive-8142bf56"}
{"level":"info","ts":"2026-11-03T21:51:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"},"tenantID":"e2e-derive-8142bf56","namespace":"openshift-ingress","gatewayName":"e2e-derive-8142bf56"}
{"level":"info","ts":"2026-11-03T21:52:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:52:05Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-explicit-a1c779"},"tenantID":"e2e-explicit-a1c779","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:52:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:52:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:52:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:52:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:52:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:52:20Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-fd831e1c"},"tenantID":"e2e-mt-fd831e1c","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:52:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:52:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-11-03T21:52:39Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rt-sw-d43634","namespace":"llm"},"namespace":"llm","name":"e2e-rt-sw-d43634","reconcileID":"325cd752-a69d-4b0c-930d-dcbe52e63e8e","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-rt-sw-d43634\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:52:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:52:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"error","ts":"2026-11-03T21:52:46Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-broken-e05b34","namespace":"e2e-extmod-cfl-2-f39f81"},"namespace":"e2e-extmod-cfl-2-f39f81","name":"e2e-cfl-broken-e05b34","reconcileID":"505f9f0b-25fc-4002-ba88-d05e33fae801","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-cfl-broken-e05b34\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:52:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"},"tenantID":"e2e-derive-8142bf56","namespace":"openshift-ingress","gatewayName":"e2e-derive-8142bf56"}
{"level":"info","ts":"2026-11-03T21:52:48Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"}}
{"level":"info","ts":"2026-11-03T21:52:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:52:51Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"}}
{"level":"info","ts":"2026-11-03T21:52:54Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-e92519"},"tenantID":"e2e-shared-a-w4-e92519","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T21:53:00Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-ok-d90829","namespace":"e2e-extmod-cfl-1-6f7706"},"namespace":"e2e-extmod-cfl-1-6f7706","name":"e2e-cfl-ok-d90829","reconcileID":"aca20aa6-8083-421e-9760-07346139c309","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-cfl-ok-d90829\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:53:00Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"}}
{"level":"info","ts":"2026-11-03T21:53:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:53:06Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-8142bf56"},"tenantID":"e2e-derive-8142bf56","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:53:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:53:17Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"}}
{"level":"info","ts":"2026-11-03T21:53:19Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-3880e7"},"tenantID":"e2e-shared-b-w4-3880e7","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:53:25Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:53:27Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:53:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-91c494"},"tenantID":"e2e-api-a-91c494","namespace":"openshift-ingress","gatewayName":"e2e-api-a-91c494"}
{"level":"info","ts":"2026-11-03T21:53:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"error","ts":"2026-11-03T21:53:37Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-cfl-same-d455b8","namespace":"e2e-extmod-cfl-2-f39f81"},"namespace":"e2e-extmod-cfl-2-f39f81","name":"e2e-cfl-same-d455b8","reconcileID":"6fe38c1c-1b19-43c3-9653-856aafb18791","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-cfl-same-d455b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:53:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-91c494"},"tenantID":"e2e-api-a-91c494","namespace":"openshift-ingress","gatewayName":"e2e-api-a-91c494"}
{"level":"info","ts":"2026-11-03T21:53:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:53:40Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:53:40Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:40Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:53:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-91c494"},"tenantID":"e2e-api-a-91c494","namespace":"openshift-ingress","gatewayName":"e2e-api-a-91c494"}
{"level":"info","ts":"2026-11-03T21:53:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:53:55Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:55Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:53:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-91c494"},"tenantID":"e2e-api-a-91c494","namespace":"openshift-ingress","gatewayName":"e2e-api-a-91c494"}
{"level":"info","ts":"2026-11-03T21:53:59Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:59Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:59Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:53:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:54:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:54:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:54:09Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:54:11Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a7f34a"},"tenantID":"e2e-worker-w0-a7f34a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:54:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:54:19Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:54:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:54:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:54:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:54:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:54:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-91c494"},"tenantID":"e2e-api-a-91c494","namespace":"openshift-ingress","gatewayName":"e2e-api-a-91c494"}
{"level":"info","ts":"2026-11-03T21:54:29Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:54:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:54:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress","gatewayName":"e2e-api-b-109220"}
{"level":"info","ts":"2026-11-03T21:54:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress","gatewayName":"e2e-api-b-109220"}
{"level":"info","ts":"2026-11-03T21:54:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:54:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:54:53Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:54:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"error","ts":"2026-11-03T21:55:03Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-dft-07ad0e","namespace":"llm"},"namespace":"llm","name":"e2e-dft-07ad0e","reconcileID":"5016dfc8-d176-4b1b-b611-eb542352b583","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-dft-07ad0e\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:55:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress","gatewayName":"e2e-api-b-109220"}
{"level":"info","ts":"2026-11-03T21:55:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:09Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:55:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress","gatewayName":"e2e-api-b-109220"}
{"level":"info","ts":"2026-11-03T21:55:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:23Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:55:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:33Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:55:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress","gatewayName":"e2e-api-b-109220"}
{"level":"info","ts":"2026-11-03T21:55:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress","gatewayName":"e2e-api-b-109220"}
{"level":"info","ts":"2026-11-03T21:55:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:55:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:43Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T21:55:44Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-dft-refs-b997cf","namespace":"llm"},"namespace":"llm","name":"e2e-dft-refs-b997cf","reconcileID":"bd058fe8-34de-4e03-842f-429fa2412565","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-dft-refs-b997cf\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:55:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-11-03T21:55:53Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-dft-del-457ff7","namespace":"llm"},"namespace":"llm","name":"e2e-dft-del-457ff7","reconcileID":"4c11384b-d4a4-4419-89ea-e7a5e349ab85","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-dft-del-457ff7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:55:54Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-a-91c494"},"tenantID":"e2e-api-a-91c494","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:55:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:55:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:55:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:55:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T21:56:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:56:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:56:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:56:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:56:15Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:56:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:56:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:56:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:56:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:56:31Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"}}
{"level":"info","ts":"2026-11-03T21:56:33Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-api-b-109220"},"tenantID":"e2e-api-b-109220","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:56:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:56:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:56:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:56:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:56:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:56:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:57:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:57:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:57:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-885382"}
{"level":"info","ts":"2026-11-03T21:57:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:57:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:57:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:57:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:10Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:57:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:57:20Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:57:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:57:30Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:57:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:57:40Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:57:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:57:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T21:57:54Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-fail-3187cf8e","namespace":"e2e-extmod-fail-user-ab75f5"},"namespace":"e2e-extmod-fail-user-ab75f5","name":"e2e-fail-3187cf8e","reconcileID":"e3c19cef-43fc-4718-a35a-84d32895d1ad","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-fail-3187cf8e\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:57:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:57:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:57:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:58:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:58:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T21:58:07Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-fail-0bb5e6da","namespace":"e2e-extmod-fail-user-ab75f5"},"namespace":"e2e-extmod-fail-user-ab75f5","name":"e2e-fail-0bb5e6da","reconcileID":"0efd9886-4aec-4d44-8afc-70395c144e54","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-fail-0bb5e6da\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:58:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:58:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:58:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:58:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:58:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:58:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:58:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:58:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:58:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:58:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:58:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:58:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:58:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:58:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:58:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:58:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:58:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:58:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:58:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:58:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:58:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:58:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:59:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:59:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:59:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:59:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:59:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:59:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:59:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T21:59:28Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"tenantID":"e2e-worker-w3-885382","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T21:59:28Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-885382"},"namespace":"ai-tenant-e2e-worker-w3-885382","name":"default-tenant","reconcileID":"3fc5fa9c-caaf-4cd5-8923-4696d4e8e8cb","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T21:59:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:59:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:59:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:59:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:59:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:59:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:59:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:59:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T21:59:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T21:59:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T21:59:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T21:59:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T21:59:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:00:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:00:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:00:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:00:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:00:05Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:00:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:00:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:00:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:00:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:00:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:00:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:00:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:00:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:00:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:00:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:00:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:00:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:00:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:00:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:00:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:00:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:00:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:00:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:00:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:00:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:00:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:00:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:01:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:01:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:01:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T22:01:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:01:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:01:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:01:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:01:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:01:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:01:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:01:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:01:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:01:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:01:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:01:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:01:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:01:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:01:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:01:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:01:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:01:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:01:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T22:01:54Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-quota-47958b","namespace":"e2e-extmod-shr-user-c44ef4"},"namespace":"e2e-extmod-shr-user-c44ef4","name":"e2e-shr-quota-47958b","reconcileID":"514d3520-3aed-49d3-a4a8-e01ea9a302e3","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-quota-47958b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T22:01:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:01:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:02:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"error","ts":"2026-11-03T22:02:02Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-amb-7c42cf","namespace":"e2e-extmod-shr-user-c44ef4"},"namespace":"e2e-extmod-shr-user-c44ef4","name":"e2e-shr-amb-7c42cf","reconcileID":"be6a1fbc-b77d-43dd-b70b-85216e186fa2","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-amb-7c42cf\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T22:02:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:02:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:02:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:02:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:02:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:02:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:02:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:02:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:02:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:02:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:02:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:02:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:02:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:02:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T22:02:27Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-auth-7ee421","namespace":"e2e-extmod-shr-user-c44ef4"},"namespace":"e2e-extmod-shr-user-c44ef4","name":"e2e-shr-auth-7ee421","reconcileID":"841e337e-a669-4d29-8119-49d5c7fcc939","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-auth-7ee421\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T22:02:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:02:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:02:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:02:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:02:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-11-03T22:02:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:02:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:02:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:02:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:02:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:02:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T22:02:53Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-shr-shared-b5efd0","namespace":"e2e-extmod-shr-user-c44ef4"},"namespace":"e2e-extmod-shr-user-c44ef4","name":"e2e-shr-shared-b5efd0","reconcileID":"92ec8924-c466-493c-9f69-ec57810e8789","error":"update ExternalModel attachment finalizer: externalmodels.inference.opendatahub.io \"e2e-shr-shared-b5efd0\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T22:02:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:02:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:03:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-a4dad5"}
{"level":"info","ts":"2026-11-03T22:03:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-8b1850"}
{"level":"info","ts":"2026-11-03T22:03:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:03:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:03:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:03:07Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T22:03:09Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"tenantID":"e2e-extmod-a-a4dad5","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-11-03T22:03:09Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-a4dad5"},"namespace":"ai-tenant-e2e-extmod-a-a4dad5","name":"default-tenant","reconcileID":"91d38cdb-3198-4ab6-8334-0f527135a269","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T22:03:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:03:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:03:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"error","ts":"2026-11-03T22:03:22Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-b-8b1850/payload-processing-e2e-extmod-b-8b1850: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-8b1850\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-8b1850 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-11-03T22:03:22Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"namespace":"ai-tenant-e2e-extmod-b-8b1850","name":"default-tenant","reconcileID":"0bcc33c4-b465-4eb2-8aed-37f03bba4adc","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-b-8b1850/payload-processing-e2e-extmod-b-8b1850: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-8b1850\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-8b1850 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-11-03T22:03:24Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-8b1850"},"tenantID":"e2e-extmod-b-8b1850","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-11-03T22:03:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:03:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:03:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}
{"level":"info","ts":"2026-11-03T22:03:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-a875b4e0"}}
{"level":"info","ts":"2026-11-03T22:03:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-dbca605d"}}
{"level":"info","ts":"2026-11-03T22:03:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-ac2b9c83"}}