{"level":"info","ts":"2026-10-30T04:03:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:03:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:03:15Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:03:18Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-e10c0f"},"tenantID":"e2e-extmod-a-e10c0f","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-e10c0f"}
{"level":"info","ts":"2026-10-30T04:03:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:03:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-372480"},"tenantID":"e2e-extmod-b-372480","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-372480"}
{"level":"info","ts":"2026-10-30T04:03:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"tenantID":"e2e-shared-a-w4-7d99a4","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-7d99a4"}
{"level":"info","ts":"2026-10-30T04:03:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:03:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:03:23Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:03:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:03:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:03:25Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:03:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-852107"}
{"level":"info","ts":"2026-10-30T04:03:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-852107"}
{"level":"info","ts":"2026-10-30T04:03:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:03:31Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:03:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:03:33Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:03:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:03:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:03:35Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:03:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:03:41Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:03:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:03:43Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:03:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-852107"}
{"level":"info","ts":"2026-10-30T04:03:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a9c9b6"},"tenantID":"e2e-worker-w0-a9c9b6","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a9c9b6"}
{"level":"info","ts":"2026-10-30T04:03:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:03:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:03:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:03:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:03:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-2d5b09"},"tenantID":"e2e-worker-w2-2d5b09","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-2d5b09"}
{"level":"info","ts":"2026-10-30T04:03:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"},"tenantID":"e2e-worker-w3-3c874f","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3c874f"}
{"level":"info","ts":"2026-10-30T04:03:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-8d80b1"},"tenantID":"e2e-worker-w4-8d80b1","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-8d80b1"}
{"level":"info","ts":"2026-10-30T04:03:51Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:03:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1597f8bc"},"tenantID":"e2e-mt-1597f8bc","namespace":"openshift-ingress","gatewayName":"e2e-mt-1597f8bc"}
{"level":"info","ts":"2026-10-30T04:03:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:03:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:03:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"tenantID":"e2e-worker-w1-e44313","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-e44313"}
{"level":"info","ts":"2026-10-30T04:03:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:03:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:03:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:03:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:03:59Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:04:00Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1597f8bc"},"tenantID":"e2e-mt-1597f8bc","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:04:01Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:04:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:04:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:04:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:04:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:04:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:04:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:04:11Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:04:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:04:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:04:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:04:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:04:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:04:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:04:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:04:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:04:21Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:04:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:04:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:04:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:04:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:04:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:04:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:04:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-441461"},"tenantID":"e2e-extmod-a-441461","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-441461"}
{"level":"info","ts":"2026-10-30T04:04:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:04:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:04:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:04:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:04:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:04:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:04:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:04:42Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:04:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:04:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:04:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:04:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:04:46Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:04:47Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-441461"},"tenantID":"e2e-extmod-a-441461","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:04:48Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:04:52Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:04:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-855e14"},"tenantID":"e2e-extmod-b-855e14","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-855e14"}
{"level":"info","ts":"2026-10-30T04:04:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:04:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:04:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:04:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:04:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:04:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:04:56Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:04:58Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:05:02Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:05:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:05:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:05:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:05:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:05:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:05:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:05:12Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:05:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:05:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:05:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:05:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:05:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"error","ts":"2026-10-30T04:05:17Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-81afe3"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-b-81afe3/payload-processing-e2e-extmod-b-81afe3: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-81afe3\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-81afe3 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:05:17Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-81afe3"},"namespace":"ai-tenant-e2e-extmod-b-81afe3","name":"default-tenant","reconcileID":"d4fa6d69-765f-45ab-9b06-cef5d25babcd","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-b-81afe3/payload-processing-e2e-extmod-b-81afe3: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-b-81afe3\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-b-81afe3 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:05:17Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-81afe3"}}
{"level":"info","ts":"2026-10-30T04:05:18Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-81afe3"},"tenantID":"e2e-extmod-b-81afe3","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:05:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:05:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-da4553"},"tenantID":"e2e-shared-a-w3-da4553","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-da4553"}
{"level":"info","ts":"2026-10-30T04:05:22Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-da4553"},"tenantID":"e2e-shared-a-w3-da4553","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w3-da4553"}
{"level":"info","ts":"2026-10-30T04:05:22Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:05:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:05:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:05:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:05:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:05:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:05:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:05:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:05:32Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:05:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-1a1b45"},"tenantID":"e2e-shared-a-w1-1a1b45","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-1a1b45"}
{"level":"info","ts":"2026-10-30T04:05:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2"},"tenantID":"e2e-shared-b-w3-e1a0c2","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w3-e1a0c2"}
{"level":"info","ts":"2026-10-30T04:05:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:05:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-1a1b45"},"tenantID":"e2e-shared-a-w1-1a1b45","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w1-1a1b45"}
{"level":"info","ts":"2026-10-30T04:05:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:05:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:05:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:05:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:05:37Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:05:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-6ad168"},"tenantID":"e2e-shared-a-w0-6ad168","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w0-6ad168"}
{"level":"info","ts":"2026-10-30T04:05:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:05:40Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-689bcf"}}
{"level":"info","ts":"2026-10-30T04:05:41Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-689bcf"},"tenantID":"e2e-extmod-a-689bcf","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:05:42Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-817531"},"tenantID":"e2e-shared-b-w0-817531","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w0-817531"}
{"level":"info","ts":"2026-10-30T04:05:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-547a78"},"tenantID":"e2e-shared-b-w1-547a78","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-547a78"}
{"level":"info","ts":"2026-10-30T04:05:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:05:44Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:05:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-547a78"},"tenantID":"e2e-shared-b-w1-547a78","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w1-547a78"}
{"level":"info","ts":"2026-10-30T04:05:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:05:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:05:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:05:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:05:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:05:47Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:05:49Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-9d8692"},"tenantID":"e2e-extmod-a-9d8692","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:05:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:05:54Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:05:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:05:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:05:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:05:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:05:57Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:05:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:06:04Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-93b821"},"tenantID":"e2e-extmod-b-93b821","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:06:04Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:06:07Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-6ad168"}}
{"level":"info","ts":"2026-10-30T04:06:08Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w0-6ad168"},"tenantID":"e2e-shared-a-w0-6ad168","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:06:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:06:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:06:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:06:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:06:08Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:06:09Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:06:14Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:06:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:06:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:06:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:06:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:06:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:06:18Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:06:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:06:19Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:06:21Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w1-1a1b45"},"tenantID":"e2e-shared-a-w1-1a1b45","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:06:24Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-35f43f"},"tenantID":"e2e-extmod-b-35f43f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:06:24Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:06:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:06:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:06:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:06:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:06:28Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:06:29Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:06:34Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:06:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:06:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:06:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:06:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:06:38Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:06:39Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:06:40Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-817531"}}
{"level":"info","ts":"2026-10-30T04:06:40Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w0-817531"},"tenantID":"e2e-shared-b-w0-817531","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:06:44Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-547a78"}}
{"level":"info","ts":"2026-10-30T04:06:45Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-547a78"},"tenantID":"e2e-shared-b-w1-547a78","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:06:45Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-547a78"},"namespace":"ai-tenant-e2e-shared-b-w1-547a78","name":"default-tenant","reconcileID":"596a8b6b-abbe-402a-a0b7-456943225eb5","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:06:45Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w1-547a78"},"tenantID":"e2e-shared-b-w1-547a78","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:06:45Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:06:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"tenantID":"e2e-shared-a-w4-7d99a4","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-7d99a4"}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"tenantID":"e2e-shared-a-w4-7d99a4","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w4-7d99a4"}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:06:49Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:06:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-852107"}
{"level":"info","ts":"2026-10-30T04:06:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:06:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w4-852107"}
{"level":"info","ts":"2026-10-30T04:06:55Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"error","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-e10c0f"},"error":"apply NetworkPolicy ai-tenant-e2e-extmod-a-e10c0f/payload-processing-e2e-extmod-a-e10c0f: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-e10c0f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-e10c0f because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:06:59Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-e10c0f"},"namespace":"ai-tenant-e2e-extmod-a-e10c0f","name":"default-tenant","reconcileID":"fcbbeefa-611f-4841-ab9a-96a537eb1dc3","error":"apply: apply NetworkPolicy ai-tenant-e2e-extmod-a-e10c0f/payload-processing-e2e-extmod-a-e10c0f: networkpolicies.networking.k8s.io \"payload-processing-e2e-extmod-a-e10c0f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-extmod-a-e10c0f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-e10c0f"},"tenantID":"e2e-extmod-a-e10c0f","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:06:59Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-e10c0f"},"namespace":"ai-tenant-e2e-extmod-a-e10c0f","name":"default-tenant","reconcileID":"b190e58e-c047-4698-b9b2-c9a400367c04","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:06:59Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:07:00Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-e10c0f"},"tenantID":"e2e-extmod-a-e10c0f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:07:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:07:05Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w1-e44313/payload-processing-e2e-worker-w1-e44313: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-e44313\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-e44313 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:05Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"namespace":"ai-tenant-e2e-worker-w1-e44313","name":"default-tenant","reconcileID":"f6f15313-a6ba-4caa-8c73-a568d4f7560b","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w1-e44313/payload-processing-e2e-worker-w1-e44313: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-e44313\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-e44313 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:06Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w1-e44313/payload-processing-e2e-worker-w1-e44313: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-e44313\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-e44313 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:06Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"namespace":"ai-tenant-e2e-worker-w1-e44313","name":"default-tenant","reconcileID":"f335e096-e406-4a2e-9dff-28533564ab58","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w1-e44313/payload-processing-e2e-worker-w1-e44313: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-e44313\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-e44313 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:07:06Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"error","ts":"2026-10-30T04:07:07Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w1-e44313/payload-processing-e2e-worker-w1-e44313: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-e44313\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-e44313 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:07Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"namespace":"ai-tenant-e2e-worker-w1-e44313","name":"default-tenant","reconcileID":"bb499ba9-deec-4631-ad02-32899e9f3f23","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w1-e44313/payload-processing-e2e-worker-w1-e44313: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w1-e44313\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w1-e44313 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:07:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:07:08Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w3-da4553"},"tenantID":"e2e-shared-a-w3-da4553","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:07:09Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-e44313"},"tenantID":"e2e-worker-w1-e44313","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:07:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:07:14Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-372480"}}
{"level":"info","ts":"2026-10-30T04:07:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-372480"},"tenantID":"e2e-extmod-b-372480","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:07:16Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:07:17Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:07:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:07:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:07:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:07:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:07:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:07:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:07:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:07:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:07:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:07:25Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a9c9b6"}}
{"level":"info","ts":"2026-10-30T04:07:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a9c9b6"},"tenantID":"e2e-worker-w0-a9c9b6","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:07:26Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:07:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:07:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:07:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:07:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:07:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:07:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:07:31Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w3-e1a0c2/payload-processing-e2e-shared-b-w3-e1a0c2: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-e1a0c2\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-e1a0c2 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:31Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2"},"namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2","name":"default-tenant","reconcileID":"1cc06282-d0f1-4ba9-abac-a7e051bcbacf","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w3-e1a0c2/payload-processing-e2e-shared-b-w3-e1a0c2: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w3-e1a0c2\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-e1a0c2 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:07:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2"},"tenantID":"e2e-shared-b-w3-e1a0c2","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:07:32Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2"},"tenantID":"e2e-shared-b-w3-e1a0c2","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:07:32Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2"},"namespace":"ai-tenant-e2e-shared-b-w3-e1a0c2","name":"default-tenant","reconcileID":"25eab975-f3a6-4ce7-b99e-3c80a87e6734","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:07:36Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"}}
{"level":"info","ts":"2026-10-30T04:07:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:07:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:07:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:07:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:07:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:07:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:07:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:07:46Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-1014c453"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:49Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-a-w4-7d99a4/payload-processing-e2e-shared-a-w4-7d99a4: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w4-7d99a4\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w4-7d99a4 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:07:49Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"namespace":"ai-tenant-e2e-shared-a-w4-7d99a4","name":"default-tenant","reconcileID":"0b246957-5062-4c08-a713-3272b2ecf9f5","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-a-w4-7d99a4/payload-processing-e2e-shared-a-w4-7d99a4: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-a-w4-7d99a4\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w4-7d99a4 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:07:49Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"}}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-398c25"},"tenantID":"e2e-extmod-a-398c25","namespace":"openshift-ingress","gatewayName":"e2e-extmod-a-398c25"}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"tenantID":"e2e-shared-a-w4-7d99a4","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:07:50Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"namespace":"ai-tenant-e2e-shared-a-w4-7d99a4","name":"default-tenant","reconcileID":"aca3c64e-d36c-46f9-becd-fd726e1ecbae","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:07:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:07:51Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w4-7d99a4"},"tenantID":"e2e-shared-a-w4-7d99a4","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:08:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:08:05Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:08:05Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"tenantID":"e2e-shared-b-w4-852107","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:08:05Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w4-852107"},"namespace":"ai-tenant-e2e-shared-b-w4-852107","name":"default-tenant","reconcileID":"374757c4-87a3-4a17-9b7a-759246c7c3ad","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:08:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:08:12Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w3-3c874f/payload-processing-e2e-worker-w3-3c874f: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-3c874f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-3c874f because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:08:12Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"},"namespace":"ai-tenant-e2e-worker-w3-3c874f","name":"default-tenant","reconcileID":"ea662854-932f-4c52-9e7e-5420e02d9baa","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w3-3c874f/payload-processing-e2e-worker-w3-3c874f: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-3c874f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-3c874f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:08:12Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-w3-3c874f/payload-processing-e2e-worker-w3-3c874f: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-3c874f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-3c874f because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:08:12Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"},"namespace":"ai-tenant-e2e-worker-w3-3c874f","name":"default-tenant","reconcileID":"27375a4b-3541-46aa-abed-b074e5f18772","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-w3-3c874f/payload-processing-e2e-worker-w3-3c874f: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-w3-3c874f\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-w3-3c874f because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:08:13Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"}}
{"level":"info","ts":"2026-10-30T04:08:14Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3c874f"},"tenantID":"e2e-worker-w3-3c874f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"}}
{"level":"info","ts":"2026-10-30T04:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:08:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:08:25Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-8d80b1"}}
{"level":"info","ts":"2026-10-30T04:08:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-8d80b1"},"tenantID":"e2e-worker-w4-8d80b1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"error","ts":"2026-10-30T04:08:30Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-e1d35baa"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:08:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:08:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:08:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-2d5b09"},"tenantID":"e2e-worker-w2-2d5b09","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-2d5b09"}
{"level":"info","ts":"2026-10-30T04:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:08:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:09:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:09:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"}}
{"level":"info","ts":"2026-10-30T04:09:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:09:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress","gatewayName":"e2e-shared-a-w2-30e3dd"}
{"level":"info","ts":"2026-10-30T04:09:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:09:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:09:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"error","ts":"2026-10-30T04:09:30Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-c86d0182"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:09:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:09:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress","gatewayName":"e2e-shared-b-w2-dd81fd"}
{"level":"info","ts":"2026-10-30T04:09:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:09:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:09:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:09:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:45Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-a-w2-30e3dd"},"tenantID":"e2e-shared-a-w2-30e3dd","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:09:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:09:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:09:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:09:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:09:54Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-855e14"},"tenantID":"e2e-extmod-b-855e14","namespace":"openshift-ingress","gatewayName":"e2e-extmod-b-855e14"}
{"level":"error","ts":"2026-10-30T04:09:59Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"error":"apply NetworkPolicy ai-tenant-e2e-shared-b-w2-dd81fd/payload-processing-e2e-shared-b-w2-dd81fd: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w2-dd81fd\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w2-dd81fd because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:09:59Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"namespace":"ai-tenant-e2e-shared-b-w2-dd81fd","name":"default-tenant","reconcileID":"f98de285-2b53-4272-b0ac-97b0c0185b35","error":"apply: apply NetworkPolicy ai-tenant-e2e-shared-b-w2-dd81fd/payload-processing-e2e-shared-b-w2-dd81fd: networkpolicies.networking.k8s.io \"payload-processing-e2e-shared-b-w2-dd81fd\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w2-dd81fd because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:10:00Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-shared-b-w2-dd81fd"},"tenantID":"e2e-shared-b-w2-dd81fd","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:10:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"}}
{"level":"info","ts":"2026-10-30T04:10:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:10:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:10:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:10:10Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-unlabeled-7b3e0921"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:10:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:10:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:10:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:10:13Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-a-398c25"},"tenantID":"e2e-extmod-a-398c25","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:10:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:10:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:10:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:10:28Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-855e14"}}
{"level":"info","ts":"2026-10-30T04:10:29Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-extmod-b-855e14"},"tenantID":"e2e-extmod-b-855e14","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:10:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:10:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:10:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:10:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:10:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"}}
{"level":"info","ts":"2026-10-30T04:10:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:10:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:10:48Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-2d5b09"}}
{"level":"info","ts":"2026-10-30T04:10:49Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-2d5b09"},"tenantID":"e2e-worker-w2-2d5b09","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:10:50Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-b2486a2a"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:10:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:10:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:11:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:11:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:11:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:11:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:11:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:11:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:11:22Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"namespace":"ai-tenant-e2e-ait-6e1eb434","name":"default-tenant","reconcileID":"0b1ef1d2-29d8-42eb-91c3-8549c432dc6c","error":"ensure finalizer: Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:11:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"tenantID":"e2e-ait-6e1eb434","namespace":"openshift-ingress","gatewayName":"e2e-ait-6e1eb434"}
{"level":"info","ts":"2026-10-30T04:11:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"tenantID":"e2e-ait-6e1eb434","namespace":"openshift-ingress","gatewayName":"e2e-ait-6e1eb434"}
{"level":"info","ts":"2026-10-30T04:11:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"tenantID":"e2e-ait-6e1eb434","namespace":"openshift-ingress","gatewayName":"e2e-ait-6e1eb434"}
{"level":"info","ts":"2026-10-30T04:11:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:11:30Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:11:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"tenantID":"e2e-ait-6e1eb434","namespace":"openshift-ingress","gatewayName":"e2e-ait-6e1eb434"}
{"level":"info","ts":"2026-10-30T04:11:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"tenantID":"e2e-ait-6e1eb434","namespace":"openshift-ingress","gatewayName":"e2e-ait-6e1eb434"}
{"level":"info","ts":"2026-10-30T04:11:37Z","logger":"tenant","msg":"MaasTenantConfig opted into praxis but owning AITenant is not DeployReady yet (missing/Failed/stale Ready stamp); will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"}}
{"level":"info","ts":"2026-10-30T04:11:38Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"}}
{"level":"info","ts":"2026-10-30T04:11:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:11:40Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:11:48Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"}}
{"level":"info","ts":"2026-10-30T04:11:48Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-6e1eb434"},"tenantID":"e2e-ait-6e1eb434","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:11:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:11:50Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:12:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:12:00Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:12:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:12:10Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"info","ts":"2026-10-30T04:12:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"}}
{"level":"info","ts":"2026-10-30T04:12:20Z","logger":"tenant","msg":"MaasTenantConfig deleting but owning AITenant is not resolvable yet; will retry cleanup","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"}}
{"level":"error","ts":"2026-10-30T04:12:30Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-mt-3876eb62"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:12:31Z","logger":"tenant","msg":"praxis-extproc cleanup exceeded deletion timeout; force-removing finalizer without confirming cleanup succeeded","maastenantconfig":{"name":"default-tenant","namespace":"e2e-mt-other-1597f8bc"},"tenantID":"","timeout":600,"stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcileDelete\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:958\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:183\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:14:44Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:14:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:15:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:15:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:15:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:15:20Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-10-30T04:15:25Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"namespace":"ai-tenant-e2e-worker-single-main-26a1d1","name":"default-tenant","reconcileID":"9ca05ca7-7b66-4b7a-ae90-b73f6461d0c8","error":"ensure finalizer: Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:15:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:15:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:15:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:15:35Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:15:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:15:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:15:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:16:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-26a1d1"}
{"level":"info","ts":"2026-10-30T04:16:30Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:16:30Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"tenantID":"e2e-worker-single-main-26a1d1","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-30T04:16:30Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-26a1d1"},"namespace":"ai-tenant-e2e-worker-single-main-26a1d1","name":"default-tenant","reconcileID":"32773052-0b14-4c37-964d-af7f01ae0008","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:17:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:17:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:18:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:18:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:18:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:18:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:19:06Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-10-30T04:20:40Z","msg":"Reconciler error","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-rotation-external-model","namespace":"models-as-a-service"},"namespace":"models-as-a-service","name":"e2e-rotation-external-model","reconcileID":"86f06174-99c8-4aba-8f52-37b1d3cee443","error":"remove ExternalModel cleanup finalizer: externalmodels.inference.opendatahub.io \"e2e-rotation-external-model\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:20:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:20:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:20:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:20:52Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:32Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:37Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:24:49Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-10-30T04:24:57Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"namespace":"ai-tenant-e2e-worker-single-main-a9dee1","name":"default-tenant","reconcileID":"eb52edf5-a234-4c0e-915b-d409ee5b4861","error":"ensure finalizer: Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:24:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:24:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:25:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:25:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:25:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:25:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:25:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"info","ts":"2026-10-30T04:25:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress","gatewayName":"e2e-worker-single-main-a9dee1"}
{"level":"error","ts":"2026-10-30T04:26:03Z","logger":"tenant","msg":"praxis-extproc apply failed for tenant; will retry","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"error":"apply NetworkPolicy ai-tenant-e2e-worker-single-main-a9dee1/payload-processing-e2e-worker-single-main-a9dee1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-single-main-a9dee1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-single-main-a9dee1 because it is being terminated","stacktrace":"github.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).reconcilePraxis\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:472\ngithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-controller/pkg/tenant/reconciler.go:190\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:223\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:481\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"error","ts":"2026-10-30T04:26:03Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"namespace":"ai-tenant-e2e-worker-single-main-a9dee1","name":"default-tenant","reconcileID":"2cce4b29-965d-4931-8ca5-d937fb10ef24","error":"apply: apply NetworkPolicy ai-tenant-e2e-worker-single-main-a9dee1/payload-processing-e2e-worker-single-main-a9dee1: networkpolicies.networking.k8s.io \"payload-processing-e2e-worker-single-main-a9dee1\" is forbidden: unable to create new content in namespace ai-tenant-e2e-worker-single-main-a9dee1 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-30T04:26:04Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-single-main-a9dee1"},"tenantID":"e2e-worker-single-main-a9dee1","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-30T04:26:11Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:26:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:26:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:26Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-30T04:28:26Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-30T04:28:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:27Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-30T04:28:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:32Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-30T04:28:33Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:55Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-30T04:28:55Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-30T04:28:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:28:56Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:29:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:29:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:29:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:29:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:30:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:30:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-30T04:30:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}