{"level":"info","ts":"2026-10-30T05:06:56Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:06:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:06:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ed8266"},"tenantID":"e2e-extmod-a-ed8266","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-ed8266"}
{"level":"info","ts":"2026-10-30T05:07:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-483bea"},"tenantID":"e2e-extmod-b-483bea","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-483bea"}
{"level":"info","ts":"2026-10-30T05:07:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:07:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:07:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:07:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:07:06Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:07:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:07:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:07:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-26b72f"},"tenantID":"e2e-inf-b-26b72f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:07:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-26b72f"},"tenantID":"e2e-inf-b-26b72f","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:07:15Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-inf-b-26b72f"},"namespace":"ai-tenant-e2e-inf-b-26b72f","name":"default-tenant","reconcileID":"b6b6e703-d16f-4346-8fcb-d72d25e74e78","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:07:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:07:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:07:16Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:07:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:07:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:07:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:07:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:07:26Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:07:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:07:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:07:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:07:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T05:07:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e6b837"}
{"level":"info","ts":"2026-10-30T05:07:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:07:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e6b837"}
{"level":"info","ts":"2026-10-30T05:07:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:07:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:07:36Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:07:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:07:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fcf42a"},"tenantID":"e2e-extmod-b-fcf42a","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-fcf42a"}
{"level":"info","ts":"2026-10-30T05:07:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:07:45Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:45Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:45Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:07:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:07:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:47Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-6378cc"},"tenantID":"e2e-extmod-a-6378cc","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-6378cc"}
{"level":"info","ts":"2026-10-30T05:07:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:07:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e6b837"}
{"level":"info","ts":"2026-10-30T05:07:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:07:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:07:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:07:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:07:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:07:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:07:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:07:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:07:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:08:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:08:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:08:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:08:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:08:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:08:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:08:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:08:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:08:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:08:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:08:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:08:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:08:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:08:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-4c2415"},"tenantID":"e2e-shared-a-w4-4c2415","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-4c2415"}
{"level":"info","ts":"2026-10-30T05:08:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-4c2415"},"tenantID":"e2e-shared-a-w4-4c2415","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-4c2415"}
{"level":"info","ts":"2026-10-30T05:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:08:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:08:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-4c2415"},"tenantID":"e2e-shared-a-w4-4c2415","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-4c2415"}
{"level":"info","ts":"2026-10-30T05:08:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:08:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:08:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:08:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:08:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"tenantID":"e2e-worker-w2-5d74b9","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-5d74b9"}
{"level":"info","ts":"2026-10-30T05:08:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"tenantID":"e2e-worker-w3-b4708a","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-b4708a"}
{"level":"info","ts":"2026-10-30T05:08:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"tenantID":"e2e-worker-w4-a2f984","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-a2f984"}
{"level":"info","ts":"2026-10-30T05:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:08:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:08:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"tenantID":"e2e-worker-w0-36bbac","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-36bbac"}
{"level":"info","ts":"2026-10-30T05:08:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"tenantID":"e2e-shared-b-w4-bbce72","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-bbce72"}
{"level":"info","ts":"2026-10-30T05:08:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:08:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:08:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:08:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:08:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:08:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-6796b1"},"tenantID":"e2e-worker-w1-6796b1","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-6796b1"}
{"level":"info","ts":"2026-10-30T05:08:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-fcd6a6"},"tenantID":"e2e-shared-a-w2-fcd6a6","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-fcd6a6"}
{"level":"info","ts":"2026-10-30T05:08:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ed8266"},"tenantID":"e2e-extmod-a-ed8266","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-ed8266"}
{"level":"error","ts":"2026-10-30T05:08:46Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ed8266"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-ed8266/payload-processing-e2e-extmod-a-ed8266: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-ed8266\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-ed8266 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:08:46Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ed8266"},"namespace":"ai-tenant-e2e-extmod-a-ed8266","name":"default-tenant","reconcileID":"4f2254fd-d7df-4aab-ae24-5f8eebb0bae1","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-ed8266/payload-processing-e2e-extmod-a-ed8266: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-ed8266\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-ed8266 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:08:46Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-ed8266"},"tenantID":"e2e-extmod-a-ed8266","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:08:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:08:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:08:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:08:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:08:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-66217b"},"tenantID":"e2e-shared-b-w2-66217b","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-66217b"}
{"level":"info","ts":"2026-10-30T05:08:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"tenantID":"e2e-shared-b-w4-bbce72","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-bbce72"}
{"level":"info","ts":"2026-10-30T05:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:08:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:08:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:08:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:08:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:08:57Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:09:01Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:01Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:01Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:09:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:09:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:09:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:09:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:09:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:09:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:09:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:09:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:09:17Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:09:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:09:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"error","ts":"2026-10-30T05:09:23Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-6378cc"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-6378cc/payload-processing-e2e-extmod-a-6378cc: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-6378cc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-6378cc because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:09:23Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-6378cc"},"namespace":"ai-tenant-e2e-extmod-a-6378cc","name":"default-tenant","reconcileID":"4f96c4cf-71f1-4483-b976-a0ca6630d077","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-6378cc/payload-processing-e2e-extmod-a-6378cc: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-6378cc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-6378cc because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:09:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-6378cc"}}
{"level":"info","ts":"2026-10-30T05:09:24Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-6378cc"},"tenantID":"e2e-extmod-a-6378cc","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:09:25Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-483bea"},"tenantID":"e2e-extmod-b-483bea","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-483bea"}
{"level":"info","ts":"2026-10-30T05:09:26Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-483bea"}}
{"level":"info","ts":"2026-10-30T05:09:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-483bea"},"tenantID":"e2e-extmod-b-483bea","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:09:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:09:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:09:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:09:27Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:09:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:09:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:09:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:09:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:09:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:09:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:09:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:09:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:09:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:09:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:09:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:09:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:09:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:09:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:09:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:09:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:09:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:09:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:09:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:09:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:10:01Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-495ebe"}
{"level":"info","ts":"2026-10-30T05:10:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:10:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:10:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:10:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:10:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:10:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:10:07Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:10:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:10:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:10:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:10:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-503804"},"tenantID":"e2e-extmod-b-503804","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:10:17Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-fcd6a6"}}
{"level":"info","ts":"2026-10-30T05:10:18Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-fcd6a6"},"tenantID":"e2e-shared-a-w2-fcd6a6","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:10:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:10:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:10:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:10:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:10:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0e964"},"tenantID":"e2e-extmod-a-c0e964","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-c0e964"}
{"level":"info","ts":"2026-10-30T05:10:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-91705f"},"tenantID":"e2e-extmod-b-91705f","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-91705f"}
{"level":"info","ts":"2026-10-30T05:10:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:10:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:10:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:10:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress","gatewayName":"e2e-mt-b5496290"}
{"level":"info","ts":"2026-10-30T05:10:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:10:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:10:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:10:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:10:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:10:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:10:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:10:33Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"}}
{"level":"info","ts":"2026-10-30T05:10:33Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b5496290"},"tenantID":"e2e-mt-b5496290","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:10:34Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-66217b"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w2-66217b/payload-processing-e2e-shared-b-w2-66217b: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w2-66217b\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w2-66217b because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:10:34Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-66217b"},"namespace":"ai-tenant-e2e-shared-b-w2-66217b","name":"default-tenant","reconcileID":"2227aefa-05be-4fae-8343-9dc5af9a3586","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w2-66217b/payload-processing-e2e-shared-b-w2-66217b: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w2-66217b\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w2-66217b because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:10:35Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-66217b"},"tenantID":"e2e-shared-b-w2-66217b","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:10:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:10:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:10:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:10:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:10:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:10:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:10:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:10:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:10:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:10:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:10:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"error","ts":"2026-10-30T05:10:55Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w2-5d74b9/payload-processing-e2e-worker-w2-5d74b9: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-5d74b9\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-5d74b9 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:10:55Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"namespace":"ai-tenant-e2e-worker-w2-5d74b9","name":"default-tenant","reconcileID":"cc1dd141-2762-4000-84eb-0e867ce0b3be","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w2-5d74b9/payload-processing-e2e-worker-w2-5d74b9: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-5d74b9\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-5d74b9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:10:56Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w2-5d74b9/payload-processing-e2e-worker-w2-5d74b9: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-5d74b9\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-5d74b9 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:10:56Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"namespace":"ai-tenant-e2e-worker-w2-5d74b9","name":"default-tenant","reconcileID":"49328b8b-dd86-4326-a6d6-81e77a2ba7eb","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w2-5d74b9/payload-processing-e2e-worker-w2-5d74b9: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w2-5d74b9\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w2-5d74b9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:10:57Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"tenantID":"e2e-worker-w2-5d74b9","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:10:57Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"tenantID":"e2e-worker-w2-5d74b9","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:10:57Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-5d74b9"},"namespace":"ai-tenant-e2e-worker-w2-5d74b9","name":"default-tenant","reconcileID":"66b18c92-7c20-418b-979b-b8f98227a2c0","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:10:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:11:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:11:03Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:11:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:11:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:11:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:11:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:11:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:11:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:11:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0d6e7"},"tenantID":"e2e-extmod-a-c0d6e7","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-c0d6e7"}
{"level":"info","ts":"2026-10-30T05:11:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:11:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:11:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:11:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:11:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:11:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:11:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:11:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-05bee8"},"tenantID":"e2e-extmod-a-05bee8","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-05bee8"}
{"level":"info","ts":"2026-10-30T05:11:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:11:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-b195d0"},"tenantID":"e2e-extmod-b-b195d0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-b195d0"}
{"level":"info","ts":"2026-10-30T05:11:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:11:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"error","ts":"2026-10-30T05:11:24Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-6796b1"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w1-6796b1/payload-processing-e2e-worker-w1-6796b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-6796b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-6796b1 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:11:24Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-6796b1"},"namespace":"ai-tenant-e2e-worker-w1-6796b1","name":"default-tenant","reconcileID":"9caf23a2-ce6b-4415-ade1-80f6b7cbe2ef","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w1-6796b1/payload-processing-e2e-worker-w1-6796b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-6796b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-6796b1 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:11:25Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-6796b1"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w1-6796b1/payload-processing-e2e-worker-w1-6796b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-6796b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-6796b1 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:11:25Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-6796b1"},"namespace":"ai-tenant-e2e-worker-w1-6796b1","name":"default-tenant","reconcileID":"9f706996-c661-4627-99a5-3409286b41d1","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w1-6796b1/payload-processing-e2e-worker-w1-6796b1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-6796b1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-6796b1 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:11:25Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-6796b1"},"tenantID":"e2e-worker-w1-6796b1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:11:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:11:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:11:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:11:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:11:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:11:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:11:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:11:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:11:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:11:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-495ebe"}
{"level":"info","ts":"2026-10-30T05:11:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-495ebe"}
{"level":"info","ts":"2026-10-30T05:11:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:11:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:11:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:11:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-4c2415"},"tenantID":"e2e-shared-a-w4-4c2415","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-4c2415"}
{"level":"info","ts":"2026-10-30T05:11:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-4c2415"},"tenantID":"e2e-shared-a-w4-4c2415","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-4c2415"}
{"level":"info","ts":"2026-10-30T05:11:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:11:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:11:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:11:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-495ebe"}
{"level":"info","ts":"2026-10-30T05:11:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:11:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:11:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:11:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:11:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:11:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:11:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:11:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"tenantID":"e2e-shared-b-w4-bbce72","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-bbce72"}
{"level":"info","ts":"2026-10-30T05:11:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"tenantID":"e2e-shared-b-w4-bbce72","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-bbce72"}
{"level":"info","ts":"2026-10-30T05:12:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:12:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:12:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"error","ts":"2026-10-30T05:12:06Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0d6e7"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-c0d6e7/payload-processing-e2e-extmod-a-c0d6e7: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-c0d6e7\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-c0d6e7 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:12:06Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0d6e7"},"namespace":"ai-tenant-e2e-extmod-a-c0d6e7","name":"default-tenant","reconcileID":"156bb0c0-5713-47f4-9fd2-3902658ea3d4","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-c0d6e7/payload-processing-e2e-extmod-a-c0d6e7: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-c0d6e7\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-c0d6e7 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:12:06Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0d6e7"},"tenantID":"e2e-extmod-a-c0d6e7","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:12:06Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0d6e7"},"namespace":"ai-tenant-e2e-extmod-a-c0d6e7","name":"default-tenant","reconcileID":"6bf24bf4-ec65-4df4-b43d-bab4c47734d8","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:12:07Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0d6e7"},"tenantID":"e2e-extmod-a-c0d6e7","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:12:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:12:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:12:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:12:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:12:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:12:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:12:13Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:12:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:12:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:12:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:12:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"error","ts":"2026-10-30T05:12:21Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fcf42a"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-b-fcf42a/payload-processing-e2e-extmod-b-fcf42a: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-fcf42a\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-fcf42a because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:12:21Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fcf42a"},"namespace":"ai-tenant-e2e-extmod-b-fcf42a","name":"default-tenant","reconcileID":"953a7fa3-fb1e-485c-83be-7a1e366c5082","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-b-fcf42a/payload-processing-e2e-extmod-b-fcf42a: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-fcf42a\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-fcf42a because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:12:22Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fcf42a"},"tenantID":"e2e-extmod-b-fcf42a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:12:23Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fcf42a"},"tenantID":"e2e-extmod-b-fcf42a","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:12:23Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-fcf42a"},"namespace":"ai-tenant-e2e-extmod-b-fcf42a","name":"default-tenant","reconcileID":"e93a5d96-9b43-4cab-8ac8-a68c470fe50d","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:12:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:12:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:12:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:12:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:12:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:12:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:12:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:12:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:12:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:12:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:12:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T05:12:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:12:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:12:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:12:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:12:43Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-4c2415"},"tenantID":"e2e-shared-a-w4-4c2415","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:12:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:12:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:12:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:12:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:12:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:12:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:12:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:12:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e6b837"}
{"level":"info","ts":"2026-10-30T05:12:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:12:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:12:53Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:12:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:12:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:12:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:12:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:13:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:13:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:13:03Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:13:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:13:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:13:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:13:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"error","ts":"2026-10-30T05:13:13Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w4-bbce72/payload-processing-e2e-shared-b-w4-bbce72: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w4-bbce72\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w4-bbce72 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:13:13Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"namespace":"ai-tenant-e2e-shared-b-w4-bbce72","name":"default-tenant","reconcileID":"2e5c4ac4-3c37-421e-a33f-23d32a78041c","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w4-bbce72/payload-processing-e2e-shared-b-w4-bbce72: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w4-bbce72\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w4-bbce72 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:13:14Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-bbce72"},"tenantID":"e2e-shared-b-w4-bbce72","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:13:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:13:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:13:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:13:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:13:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:13:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:13:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:13:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:13:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:13:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:13:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:13:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e6b837"}
{"level":"info","ts":"2026-10-30T05:13:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-e6b837"}
{"level":"info","ts":"2026-10-30T05:13:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:13:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:13:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:13:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:13:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:13:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-247158"}
{"level":"info","ts":"2026-10-30T05:13:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"tenantID":"e2e-worker-w3-b4708a","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-b4708a"}
{"level":"info","ts":"2026-10-30T05:13:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"tenantID":"e2e-worker-w4-a2f984","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-a2f984"}
{"level":"info","ts":"2026-10-30T05:13:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:13:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:13:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:13:35Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-c0e964"},"tenantID":"e2e-extmod-a-c0e964","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:13:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"tenantID":"e2e-worker-w0-36bbac","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-36bbac"}
{"level":"info","ts":"2026-10-30T05:13:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:13:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:13:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:13:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:13:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:13:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:13:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:13:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:13:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:13:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:13:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"error","ts":"2026-10-30T05:13:50Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w4-a2f984/payload-processing-e2e-worker-w4-a2f984: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-a2f984\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-a2f984 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:13:50Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"namespace":"ai-tenant-e2e-worker-w4-a2f984","name":"default-tenant","reconcileID":"6270736f-6c2d-4f15-886b-49db43c26e84","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w4-a2f984/payload-processing-e2e-worker-w4-a2f984: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-a2f984\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-a2f984 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:13:51Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w4-a2f984/payload-processing-e2e-worker-w4-a2f984: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-a2f984\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-a2f984 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:13:51Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"namespace":"ai-tenant-e2e-worker-w4-a2f984","name":"default-tenant","reconcileID":"aeb4670d-6f9e-4c04-8bc8-dd9f769c55e2","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w4-a2f984/payload-processing-e2e-worker-w4-a2f984: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w4-a2f984\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w4-a2f984 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:13:51Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-a2f984"},"tenantID":"e2e-worker-w4-a2f984","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:13:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:13:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-e070ea"}
{"level":"info","ts":"2026-10-30T05:13:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:13:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:13:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:13:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-495ebe"}
{"level":"info","ts":"2026-10-30T05:13:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-495ebe"}
{"level":"info","ts":"2026-10-30T05:13:57Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-91705f"},"tenantID":"e2e-extmod-b-91705f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:13:57Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-91705f"},"tenantID":"e2e-extmod-b-91705f","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:13:57Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-91705f"},"namespace":"ai-tenant-e2e-extmod-b-91705f","name":"default-tenant","reconcileID":"3edea444-a6f4-4d1e-a405-e6ca1bd34bdf","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:13:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:13:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:13:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:13:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"error","ts":"2026-10-30T05:14:02Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-05bee8"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-05bee8/payload-processing-e2e-extmod-a-05bee8: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-05bee8\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-05bee8 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:14:02Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-05bee8"},"namespace":"ai-tenant-e2e-extmod-a-05bee8","name":"default-tenant","reconcileID":"921f219f-f594-4d61-96fe-6b7d598d4eda","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-05bee8/payload-processing-e2e-extmod-a-05bee8: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-05bee8\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-05bee8 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:14:02Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-05bee8"},"tenantID":"e2e-extmod-a-05bee8","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:14:02Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-05bee8"},"namespace":"ai-tenant-e2e-extmod-a-05bee8","name":"default-tenant","reconcileID":"ed195e4b-9b67-426e-84b0-34cb3c820e86","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:14:03Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-05bee8"},"tenantID":"e2e-extmod-a-05bee8","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:14:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:14:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:14:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:14:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:14:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:14:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:14:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:14:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:14:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"}}
{"level":"info","ts":"2026-10-30T05:14:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:14:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-b195d0"},"tenantID":"e2e-extmod-b-b195d0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-b195d0"}
{"level":"info","ts":"2026-10-30T05:14:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-b195d0"},"tenantID":"e2e-extmod-b-b195d0","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-b195d0"}
{"level":"info","ts":"2026-10-30T05:14:18Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-b195d0"}}
{"level":"info","ts":"2026-10-30T05:14:19Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-b195d0"},"tenantID":"e2e-extmod-b-b195d0","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:14:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:14:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:14:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:14:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:14:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"error","ts":"2026-10-30T05:14:24Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4877f558"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:14:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:14:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:14:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:14:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:14:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"error","ts":"2026-10-30T05:14:30Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w0-e6b837/payload-processing-e2e-shared-a-w0-e6b837: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w0-e6b837\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w0-e6b837 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:14:30Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"namespace":"ai-tenant-e2e-shared-a-w0-e6b837","name":"default-tenant","reconcileID":"3aa7ec3a-2f6a-406b-a015-d825cc106d3d","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w0-e6b837/payload-processing-e2e-shared-a-w0-e6b837: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w0-e6b837\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w0-e6b837 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:14:31Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w0-e6b837/payload-processing-e2e-shared-a-w0-e6b837: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w0-e6b837\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w0-e6b837 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:14:31Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"namespace":"ai-tenant-e2e-shared-a-w0-e6b837","name":"default-tenant","reconcileID":"e595c56a-a85b-4e30-a569-2ce5573a476b","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w0-e6b837/payload-processing-e2e-shared-a-w0-e6b837: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w0-e6b837\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w0-e6b837 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:14:31Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"}}
{"level":"info","ts":"2026-10-30T05:14:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-e6b837"},"tenantID":"e2e-shared-a-w0-e6b837","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:14:34Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-e070ea"},"tenantID":"e2e-shared-a-w3-e070ea","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:14:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:14:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:14:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:14:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:14:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:14:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:14:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:14:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:14:46Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"}}
{"level":"info","ts":"2026-10-30T05:14:47Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-247158"},"tenantID":"e2e-shared-b-w0-247158","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:14:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:14:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"}}
{"level":"info","ts":"2026-10-30T05:14:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:14:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:14:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:14:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"error","ts":"2026-10-30T05:14:56Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w3-495ebe/payload-processing-e2e-shared-b-w3-495ebe: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-495ebe\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-495ebe because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:14:56Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"namespace":"ai-tenant-e2e-shared-b-w3-495ebe","name":"default-tenant","reconcileID":"7063f027-1cc2-469e-aee6-6105f322ea5a","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w3-495ebe/payload-processing-e2e-shared-b-w3-495ebe: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-495ebe\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-495ebe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:14:57Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-495ebe"},"tenantID":"e2e-shared-b-w3-495ebe","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:14:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"error","ts":"2026-10-30T05:14:59Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-6d0a9621"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:14:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:14:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:15:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:15:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:15:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:15:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:15:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"error","ts":"2026-10-30T05:15:14Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w0-36bbac/payload-processing-e2e-worker-w0-36bbac: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-36bbac\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-36bbac because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:15:14Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"namespace":"ai-tenant-e2e-worker-w0-36bbac","name":"default-tenant","reconcileID":"e6dac3b7-5b24-4cc9-84b9-d2b44ef87d9e","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w0-36bbac/payload-processing-e2e-worker-w0-36bbac: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-36bbac\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-36bbac because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:15:14Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w0-36bbac/payload-processing-e2e-worker-w0-36bbac: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-36bbac\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-36bbac because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:15:14Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"namespace":"ai-tenant-e2e-worker-w0-36bbac","name":"default-tenant","reconcileID":"e27f9b55-bfcc-47fd-ac09-27cd856226db","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w0-36bbac/payload-processing-e2e-worker-w0-36bbac: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w0-36bbac\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w0-36bbac because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:15:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:15:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:15:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-36bbac"},"tenantID":"e2e-worker-w0-36bbac","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:15:17Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w3-b4708a/payload-processing-e2e-worker-w3-b4708a: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-b4708a\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-b4708a because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T05:15:17Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"namespace":"ai-tenant-e2e-worker-w3-b4708a","name":"default-tenant","reconcileID":"9cb411e1-0d9e-456d-a171-1a1c15d240c7","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w3-b4708a/payload-processing-e2e-worker-w3-b4708a: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-b4708a\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-b4708a because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:15:18Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"tenantID":"e2e-worker-w3-b4708a","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T05:15:18Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"tenantID":"e2e-worker-w3-b4708a","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T05:15:18Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-b4708a"},"namespace":"ai-tenant-e2e-worker-w3-b4708a","name":"default-tenant","reconcileID":"b3d26253-5bfd-438f-94f1-7535be6d1426","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T05:15:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:15:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:15:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:15:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:15:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}
{"level":"info","ts":"2026-10-30T05:15:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-458d4a40"}}
{"level":"info","ts":"2026-10-30T05:15:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-4c027df7"}}
{"level":"info","ts":"2026-10-30T05:15:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-023b5d1c"}}
{"level":"info","ts":"2026-10-30T05:15:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-15a31f77"}}
{"level":"info","ts":"2026-10-30T05:15:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-b5496290"}}