2026-07-08T18:51:18Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f2290f7f-898c-4b14-8656-67c1cd66b430", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-347a63a8 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:51:18Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "odh-trusted-ca-bundle", "reconcileID": "c8e79b33-d6d4-449a-b796-35a71f6b26ee", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-347a63a8"} 2026-07-08T18:51:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "odh-trusted-ca-bundle", "reconcileID": "c8e79b33-d6d4-449a-b796-35a71f6b26ee", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-347a63a8"} 2026-07-08T18:51:18Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "openshift-service-ca.crt", "reconcileID": "bb9e1f6a-dbde-4091-8514-f917c68ec97a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-347a63a8"} 2026-07-08T18:51:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "openshift-service-ca.crt", "reconcileID": "bb9e1f6a-dbde-4091-8514-f917c68ec97a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-347a63a8"} 2026-07-08T18:51:18Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e21b1d13-c8ff-4a75-9aef-74e818c44546", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-347a63a8"} 2026-07-08T18:51:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-347a63a8"}, "namespace": "ai-tenant-e2e-mt-347a63a8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e21b1d13-c8ff-4a75-9aef-74e818c44546", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-347a63a8"} 2026-07-08T18:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "openshift-service-ca.crt", "reconcileID": "f5b95f03-1849-419e-854f-0155c6868333", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-trusted-ca-bundle", "reconcileID": "14486849-a497-445f-903a-c90f8de303a6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "openshift-service-ca.crt", "reconcileID": "2b44653f-2e21-417f-96a2-7d6e48b61962", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f333e15d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:19Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "openshift-service-ca.crt", "reconcileID": "2b44653f-2e21-417f-96a2-7d6e48b61962", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "22ad85da-e5a0-4fd0-ad32-a7a7717afd43", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "openshift-service-ca.crt", "reconcileID": "abbd6916-5ff8-4537-b7ca-e0546d59367c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-trusted-ca-bundle", "reconcileID": "6af42da8-6dee-47f2-8dc3-0035b732f8a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f333e15d", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:19Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-trusted-ca-bundle", "reconcileID": "6af42da8-6dee-47f2-8dc3-0035b732f8a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c811ea3f-ada5-4ffa-8dab-cac2c3ca6523", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "14243518-4a65-4d6f-8ae7-a008c3c3db5d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "14243518-4a65-4d6f-8ae7-a008c3c3db5d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f333e15d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:40Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "14243518-4a65-4d6f-8ae7-a008c3c3db5d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-f333e15d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:51:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-trusted-ca-bundle", "reconcileID": "31a009c8-6309-41c9-ae03-846d59c0de3a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-trusted-ca-bundle", "reconcileID": "31a009c8-6309-41c9-ae03-846d59c0de3a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "openshift-service-ca.crt", "reconcileID": "e2cacf4d-a21c-4ffe-a04c-29d6d9157f1d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "openshift-service-ca.crt", "reconcileID": "e2cacf4d-a21c-4ffe-a04c-29d6d9157f1d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aa6716ee-db19-463f-8004-7013c5b237e7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f333e15d"}, "namespace": "e2e-unlabeled-f333e15d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aa6716ee-db19-463f-8004-7013c5b237e7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f333e15d"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "openshift-service-ca.crt", "reconcileID": "8a93c343-e660-46ff-a519-ff920a6b3d3d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-trusted-ca-bundle", "reconcileID": "5b4f78ce-fae3-4cdb-92aa-43a2f756a131", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "openshift-service-ca.crt", "reconcileID": "b7b79aee-0ce1-48c9-8feb-1cd4177348e3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c1b64785", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "openshift-service-ca.crt", "reconcileID": "b7b79aee-0ce1-48c9-8feb-1cd4177348e3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2e919f29-986a-4a1d-9328-6301ba48eea2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "openshift-service-ca.crt", "reconcileID": "eb4af26f-ec8c-46fa-bfbf-36c962c1917a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-trusted-ca-bundle", "reconcileID": "d537a228-c822-4121-a7cf-323ea5133166", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-trusted-ca-bundle", "reconcileID": "d537a228-c822-4121-a7cf-323ea5133166", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a3cdc0db-b7cd-47e0-8818-cbf7be2aabaf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "68bb5819-21c0-45b8-8942-0faebb0a960c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "68bb5819-21c0-45b8-8942-0faebb0a960c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:58Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "68bb5819-21c0-45b8-8942-0faebb0a960c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c1b64785 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:51:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-trusted-ca-bundle", "reconcileID": "0cac1d3c-30ee-40d2-a81c-3a44e102eecc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-trusted-ca-bundle", "reconcileID": "0cac1d3c-30ee-40d2-a81c-3a44e102eecc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "openshift-service-ca.crt", "reconcileID": "0976d1d2-a034-49b5-8f72-8a785efe9c5b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "openshift-service-ca.crt", "reconcileID": "0976d1d2-a034-49b5-8f72-8a785efe9c5b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b4430630-b7fa-47f3-80f8-e6fdfb943045", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c1b64785"}, "namespace": "ai-tenant-e2e-mt-c1b64785", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b4430630-b7fa-47f3-80f8-e6fdfb943045", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c1b64785"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "openshift-service-ca.crt", "reconcileID": "9f2d9e07-a5a5-4c90-865c-1614b8267c4b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-trusted-ca-bundle", "reconcileID": "600aa417-51ec-4bc3-9901-c8feed40ffbb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "openshift-service-ca.crt", "reconcileID": "b29f9029-2e1b-4f2f-a50e-ce2ebc7cf2e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2fce69f0", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "openshift-service-ca.crt", "reconcileID": "b29f9029-2e1b-4f2f-a50e-ce2ebc7cf2e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1b552f9e-dc40-4d55-863e-e41e60bd7d7a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "openshift-service-ca.crt", "reconcileID": "b0c4b629-4dfe-4b3b-adaa-4ab2252a86bd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-trusted-ca-bundle", "reconcileID": "972f1de5-92e2-4d1e-a44c-60e00a3e6387", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-trusted-ca-bundle", "reconcileID": "972f1de5-92e2-4d1e-a44c-60e00a3e6387", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b31f883e-b5b5-4870-9a47-9e3c8cdf1e8a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:51:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "openshift-service-ca.crt", "reconcileID": "78aadc40-e2c8-4d57-8959-a264332624d0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:51:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-trusted-ca-bundle", "reconcileID": "f9cb44b4-a961-45bc-9977-88799d3e8cf7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:51:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "openshift-service-ca.crt", "reconcileID": "afaa98f0-2228-43c6-ae81-f9bbb4ffa720", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bd3dee2b", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "openshift-service-ca.crt", "reconcileID": "afaa98f0-2228-43c6-ae81-f9bbb4ffa720", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:51:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3192d636-a1e7-406c-87a2-3203e39fa611", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:51:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "openshift-service-ca.crt", "reconcileID": "9608149f-ea74-4344-989e-f1d7e24342fd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:51:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-trusted-ca-bundle", "reconcileID": "be09251b-e293-4bc8-a77c-70f0829da17e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:51:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-trusted-ca-bundle", "reconcileID": "be09251b-e293-4bc8-a77c-70f0829da17e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:51:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f5d16ffd-8b5c-4c90-9ec2-47032455a5a9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "35ccab42-0862-4d07-a286-9a4ef1b8b1bc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "35ccab42-0862-4d07-a286-9a4ef1b8b1bc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:06Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "35ccab42-0862-4d07-a286-9a4ef1b8b1bc", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-2fce69f0 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-trusted-ca-bundle", "reconcileID": "a3c58be3-9f11-42aa-a519-8e02461b944c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-trusted-ca-bundle", "reconcileID": "a3c58be3-9f11-42aa-a519-8e02461b944c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "openshift-service-ca.crt", "reconcileID": "86e180c8-2c5c-4639-ab7b-7ddc9b85856a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "openshift-service-ca.crt", "reconcileID": "86e180c8-2c5c-4639-ab7b-7ddc9b85856a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc97472e-ae63-4af9-81f7-7de1a1457de8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2fce69f0"}, "namespace": "ai-tenant-e2e-mt-2fce69f0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc97472e-ae63-4af9-81f7-7de1a1457de8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2fce69f0"} 2026-07-08T18:52:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e63f27fd-bad3-4b9a-a336-26ca4b6b3b8a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e63f27fd-bad3-4b9a-a336-26ca4b6b3b8a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e63f27fd-bad3-4b9a-a336-26ca4b6b3b8a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-bd3dee2b because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-trusted-ca-bundle", "reconcileID": "2011c771-e6d0-4869-9a21-1c507f220c80", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-trusted-ca-bundle", "reconcileID": "2011c771-e6d0-4869-9a21-1c507f220c80", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "openshift-service-ca.crt", "reconcileID": "3622e66f-4849-4b2f-8311-a0eab0ed5607", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "openshift-service-ca.crt", "reconcileID": "3622e66f-4849-4b2f-8311-a0eab0ed5607", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ae0e76a7-2af4-4632-87b6-97634daafbb7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bd3dee2b"}, "namespace": "ai-tenant-e2e-mt-bd3dee2b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ae0e76a7-2af4-4632-87b6-97634daafbb7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bd3dee2b"} 2026-07-08T18:52:13Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "230e9338-834c-457e-9f01-f02b0b9fc5af", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:13Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "230e9338-834c-457e-9f01-f02b0b9fc5af", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "230e9338-834c-457e-9f01-f02b0b9fc5af", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:13Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "e768eb8f-9ccc-43c9-a93b-9045224d3d4e", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:13Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "e768eb8f-9ccc-43c9-a93b-9045224d3d4e", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:13Z ERROR Reconciler error {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "e768eb8f-9ccc-43c9-a93b-9045224d3d4e", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-mt-13d4aaa8-authn-ssl: envoyfilters.networking.istio.io \"e2e-mt-13d4aaa8-authn-ssl\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:13Z DEBUG events Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-mt-13d4aaa8-authn-ssl: envoyfilters.networking.istio.io "e2e-mt-13d4aaa8-authn-ssl" already exists {"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-mt-13d4aaa8","uid":"34d0a80b-5df6-4f31-9977-ee3f1c65a400","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"46924"}, "reason": "ReconcileError"} 2026-07-08T18:52:13Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "190fb410-5770-48f2-a799-d38126f683fa", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:13Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "190fb410-5770-48f2-a799-d38126f683fa", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "190fb410-5770-48f2-a799-d38126f683fa", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:13Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "4c3271b5-bae4-41c8-b13c-a6d3e98c972e", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:13Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "4c3271b5-bae4-41c8-b13c-a6d3e98c972e", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "4c3271b5-bae4-41c8-b13c-a6d3e98c972e", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "aaca2ee0-a137-49b8-ba76-f98625acc8e0", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "aaca2ee0-a137-49b8-ba76-f98625acc8e0", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "aaca2ee0-a137-49b8-ba76-f98625acc8e0", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "ef7434dd-c047-4820-8f8a-dfc4ff3ee6e5", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "ef7434dd-c047-4820-8f8a-dfc4ff3ee6e5", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "ef7434dd-c047-4820-8f8a-dfc4ff3ee6e5", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "f8214e62-0350-4322-b692-5c0a68d18358", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "f8214e62-0350-4322-b692-5c0a68d18358", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "f8214e62-0350-4322-b692-5c0a68d18358", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "d91db7fe-18ae-49c6-a97f-5bc1ff323d46", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "a0930a31-433c-4205-bfe7-2f2e52fca2b9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "df31fa02-4b70-4dc3-8839-35213c88daba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-13d4aaa8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "df31fa02-4b70-4dc3-8839-35213c88daba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "c909547c-29b4-4693-af49-a9adeb039bbf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf30d2fb-52b9-432e-8992-0c92a0b36dc4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "4773f0d2-417f-4914-8914-9bf4adc4cc7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "4773f0d2-417f-4914-8914-9bf4adc4cc7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "40612c88-572f-468b-bdf5-d32a2a913b21", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "42793c73-b820-4538-8fe4-7d0b68d12463", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "42793c73-b820-4538-8fe4-7d0b68d12463", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "42793c73-b820-4538-8fe4-7d0b68d12463", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "5905f4e4-84fa-4d0d-adbf-2e35599facc8", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T18:52:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "2122f1da-be7e-4fe1-986c-1e4faff5cbbf", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:22Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "1046d9e6-8d68-46bb-9917-68fc1257f666", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-13d4aaa8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:22Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "1046d9e6-8d68-46bb-9917-68fc1257f666", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:22Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "add51da5-e4a1-43b9-8419-de511f507172", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:22Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "add51da5-e4a1-43b9-8419-de511f507172", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "355fc9d0-9216-4312-b29f-cb9a8b17886c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9899201d-e145-45ee-8d5f-43a315b2a7d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "35ea6a27-766c-43bc-aa46-f5d453e73489", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:22Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "00125104-e369-4513-9e07-25309b26490b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:22Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "00125104-e369-4513-9e07-25309b26490b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9f1307c8-d4e6-4213-b905-fd1cbf7ef6ca", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "40c95ee7-af76-4960-ba40-bfb464f7d57c", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "40c95ee7-af76-4960-ba40-bfb464f7d57c", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "40c95ee7-af76-4960-ba40-bfb464f7d57c", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b34f07f8-e423-4f00-b009-3598825c111c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b34f07f8-e423-4f00-b009-3598825c111c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:28Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b34f07f8-e423-4f00-b009-3598825c111c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-other-13d4aaa8 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "2147141a-a763-4ffe-a86b-242f5cb444c1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "2147141a-a763-4ffe-a86b-242f5cb444c1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "f031b7a2-67d0-47b7-b181-f3ffcf112bd4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "f031b7a2-67d0-47b7-b181-f3ffcf112bd4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc9fa78d-bbca-4511-9c68-843a81e8afa2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-13d4aaa8"}, "namespace": "e2e-mt-other-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc9fa78d-bbca-4511-9c68-843a81e8afa2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-13d4aaa8"} 2026-07-08T18:52:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "4a18d444-4211-4f31-a65c-75e8e00243f9", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "4a18d444-4211-4f31-a65c-75e8e00243f9", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8-authn-ssl"} 2026-07-08T18:52:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-13d4aaa8","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-13d4aaa8", "reconcileID": "4a18d444-4211-4f31-a65c-75e8e00243f9", "Gateway": "e2e-mt-13d4aaa8", "namespace": "openshift-ingress"} 2026-07-08T18:52:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3fc4b179-7c00-48a7-93f9-7cb7ca89585d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3fc4b179-7c00-48a7-93f9-7cb7ca89585d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:34Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3fc4b179-7c00-48a7-93f9-7cb7ca89585d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-13d4aaa8 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "ec0e94dd-1520-439e-90dc-874fb2f81a9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-trusted-ca-bundle", "reconcileID": "ec0e94dd-1520-439e-90dc-874fb2f81a9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "d5a9f3d1-3abe-4d4f-9c13-bd4fd3b20783", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "openshift-service-ca.crt", "reconcileID": "d5a9f3d1-3abe-4d4f-9c13-bd4fd3b20783", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a796a9fe-6ad3-4dff-aa22-821f0a504d8e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-13d4aaa8"}, "namespace": "ai-tenant-e2e-mt-13d4aaa8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a796a9fe-6ad3-4dff-aa22-821f0a504d8e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-13d4aaa8"} 2026-07-08T18:52:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "openshift-service-ca.crt", "reconcileID": "e4d98938-8838-4569-b85f-92c33c948b46", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-trusted-ca-bundle", "reconcileID": "1f3d0776-8d06-4109-a1ea-7177a7fa3783", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "openshift-service-ca.crt", "reconcileID": "dc27356c-760c-4f15-822a-3aa9992ab3e3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-79251cf5", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:36Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "openshift-service-ca.crt", "reconcileID": "dc27356c-760c-4f15-822a-3aa9992ab3e3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4899503b-a312-4a0a-9803-64664b053d5a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "openshift-service-ca.crt", "reconcileID": "0b9fbc22-2c75-4156-8263-2fed6afca643", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-trusted-ca-bundle", "reconcileID": "7e43ac57-5285-46e2-a19e-1d8907c9ecfd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:36Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-trusted-ca-bundle", "reconcileID": "7e43ac57-5285-46e2-a19e-1d8907c9ecfd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "09e240e1-554a-46fa-82a5-d1091f937849", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9f07d36-59ad-4340-8281-9a9165460788", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9f07d36-59ad-4340-8281-9a9165460788", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:41Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9f07d36-59ad-4340-8281-9a9165460788", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-79251cf5 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-trusted-ca-bundle", "reconcileID": "eb97bc3f-a783-441f-8ac2-37e552f01397", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-trusted-ca-bundle", "reconcileID": "eb97bc3f-a783-441f-8ac2-37e552f01397", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "openshift-service-ca.crt", "reconcileID": "bbe0f84b-e4e9-4fa0-9b91-bf2c692efe97", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "openshift-service-ca.crt", "reconcileID": "bbe0f84b-e4e9-4fa0-9b91-bf2c692efe97", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "86e773d4-7a0e-4c1c-b208-6b20aff5ddbe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-79251cf5"}, "namespace": "e2e-webhook-sub-79251cf5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "86e773d4-7a0e-4c1c-b208-6b20aff5ddbe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-79251cf5"} 2026-07-08T18:52:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "openshift-service-ca.crt", "reconcileID": "5b763dd9-05d2-455f-9f5e-c8df68b475c4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-trusted-ca-bundle", "reconcileID": "4116a296-946b-4600-908e-0d45d485f091", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "openshift-service-ca.crt", "reconcileID": "307f1fd9-968d-4dae-8fb1-3de5c61c754a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1eb43c4a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:42Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "openshift-service-ca.crt", "reconcileID": "307f1fd9-968d-4dae-8fb1-3de5c61c754a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b9e33c09-67e5-4399-bfdd-a2d7142cbb69", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "openshift-service-ca.crt", "reconcileID": "f3eeb5d2-83d6-449f-80d6-ce4cbffbc4f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-trusted-ca-bundle", "reconcileID": "d3447e48-c9ab-47eb-bc32-52613c167470", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:42Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-trusted-ca-bundle", "reconcileID": "d3447e48-c9ab-47eb-bc32-52613c167470", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dd735280-144c-4765-954b-b54e7272ecaf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9556a6c9-9a2f-4788-8d78-bf0a7bde1ea0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9556a6c9-9a2f-4788-8d78-bf0a7bde1ea0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:47Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9556a6c9-9a2f-4788-8d78-bf0a7bde1ea0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-1eb43c4a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:52:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-trusted-ca-bundle", "reconcileID": "f523c0d8-dacc-4cfc-aba8-08d006c32fb7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-trusted-ca-bundle", "reconcileID": "f523c0d8-dacc-4cfc-aba8-08d006c32fb7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "openshift-service-ca.crt", "reconcileID": "393dbd22-840b-40b3-8a62-c0d2ece0e1ac", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "openshift-service-ca.crt", "reconcileID": "393dbd22-840b-40b3-8a62-c0d2ece0e1ac", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2dc756e3-17d3-4ded-8648-6f403a7ce86e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1eb43c4a"}, "namespace": "e2e-webhook-auth-1eb43c4a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2dc756e3-17d3-4ded-8648-6f403a7ce86e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1eb43c4a"} 2026-07-08T18:52:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "ca86f4cf-8db5-408a-9aa2-dff079f02ca9", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "ca86f4cf-8db5-408a-9aa2-dff079f02ca9", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "ca86f4cf-8db5-408a-9aa2-dff079f02ca9", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "13cb13dc-365c-4a95-bc28-433c05875d4d", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "13cb13dc-365c-4a95-bc28-433c05875d4d", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "13cb13dc-365c-4a95-bc28-433c05875d4d", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "ae7cf656-c3ae-4526-ba7f-09aaa2dd131f", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "ae7cf656-c3ae-4526-ba7f-09aaa2dd131f", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "ae7cf656-c3ae-4526-ba7f-09aaa2dd131f", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "e1e0186f-db1e-40ed-9d7d-df77380c52ba", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "e1e0186f-db1e-40ed-9d7d-df77380c52ba", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "e1e0186f-db1e-40ed-9d7d-df77380c52ba", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "c07a0b9d-af2c-495f-9ae3-6b508225009b", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "c07a0b9d-af2c-495f-9ae3-6b508225009b", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "c07a0b9d-af2c-495f-9ae3-6b508225009b", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "7605dfa6-2454-4cb0-ad1f-a8a652bcd89f", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "7605dfa6-2454-4cb0-ad1f-a8a652bcd89f", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "7605dfa6-2454-4cb0-ad1f-a8a652bcd89f", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "f0372392-d4e2-4d93-b731-63a8a85c8d9d", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:56Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "f0372392-d4e2-4d93-b731-63a8a85c8d9d", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "f0372392-d4e2-4d93-b731-63a8a85c8d9d", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "openshift-service-ca.crt", "reconcileID": "61bb279c-cea9-4896-86a0-52948cd90951", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-trusted-ca-bundle", "reconcileID": "241915e3-4479-454a-a1ef-14050d6e4a55", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "openshift-service-ca.crt", "reconcileID": "d0caf2d7-83a8-4802-8642-961b9b99b2ff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2046dd63", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "openshift-service-ca.crt", "reconcileID": "d0caf2d7-83a8-4802-8642-961b9b99b2ff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "openshift-service-ca.crt", "reconcileID": "81c266d6-559b-4f5d-9c7a-da58551b3013", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c5bd3ec4-c521-4109-bbaf-e106d496cc33", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-trusted-ca-bundle", "reconcileID": "fb6d3c54-ce62-4bb4-90f0-1cdde8907c32", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:52:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-trusted-ca-bundle", "reconcileID": "fb6d3c54-ce62-4bb4-90f0-1cdde8907c32", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1d3f2514-be37-4a11-b0ed-7cd4db1718b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:52:59Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "b718b452-fec8-47d0-8175-77f3cdf5160a", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:59Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "b718b452-fec8-47d0-8175-77f3cdf5160a", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:52:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "b718b452-fec8-47d0-8175-77f3cdf5160a", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:52:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "e1fc6a82-36c8-4421-8f2d-1138e5537f91", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T18:53:03Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "524cf86c-a8a0-404f-8359-c787367bf658", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "524cf86c-a8a0-404f-8359-c787367bf658", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "524cf86c-a8a0-404f-8359-c787367bf658", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "a4d82b94-5587-4d57-8ca1-e04964de65fb", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:03Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "524cf86c-a8a0-404f-8359-c787367bf658", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:53:03Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "524cf86c-a8a0-404f-8359-c787367bf658", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "authType": "userdefined"} 2026-07-08T18:53:03Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "524cf86c-a8a0-404f-8359-c787367bf658", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "a4d82b94-5587-4d57-8ca1-e04964de65fb", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:53:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "a4d82b94-5587-4d57-8ca1-e04964de65fb", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:03Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "3d09a77b-b8b8-41d2-8c7b-4d31cc9c5167", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "3d09a77b-b8b8-41d2-8c7b-4d31cc9c5167", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "3d09a77b-b8b8-41d2-8c7b-4d31cc9c5167", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:03Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "3d09a77b-b8b8-41d2-8c7b-4d31cc9c5167", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:53:03Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "3d09a77b-b8b8-41d2-8c7b-4d31cc9c5167", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "authType": "userdefined"} 2026-07-08T18:53:03Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "3d09a77b-b8b8-41d2-8c7b-4d31cc9c5167", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "9b16225d-f6b5-40b3-a321-644c12cb6c03", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "9b16225d-f6b5-40b3-a321-644c12cb6c03", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:53:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "9b16225d-f6b5-40b3-a321-644c12cb6c03", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:12Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "3a6b143a-0721-4b36-ab43-fa6d72de5437", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:12Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "3a6b143a-0721-4b36-ab43-fa6d72de5437", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:53:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "3a6b143a-0721-4b36-ab43-fa6d72de5437", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:14Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "386fc9f3-2524-4dff-90d9-7de0fbc531a8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:14Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "386fc9f3-2524-4dff-90d9-7de0fbc531a8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:14Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "386fc9f3-2524-4dff-90d9-7de0fbc531a8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:14Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "386fc9f3-2524-4dff-90d9-7de0fbc531a8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:53:14Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "386fc9f3-2524-4dff-90d9-7de0fbc531a8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "authType": "userdefined"} 2026-07-08T18:53:14Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "386fc9f3-2524-4dff-90d9-7de0fbc531a8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:32Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "4e6b1a8c-8ac6-4c5a-82d5-d17c2642ad1c", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:32Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "4e6b1a8c-8ac6-4c5a-82d5-d17c2642ad1c", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:32Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "4e6b1a8c-8ac6-4c5a-82d5-d17c2642ad1c", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:32Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "4e6b1a8c-8ac6-4c5a-82d5-d17c2642ad1c", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:53:32Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "4e6b1a8c-8ac6-4c5a-82d5-d17c2642ad1c", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "authType": "userdefined"} 2026-07-08T18:53:32Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "4e6b1a8c-8ac6-4c5a-82d5-d17c2642ad1c", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "6f417e1d-123a-4bff-8a0c-bf146921f242", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "6f417e1d-123a-4bff-8a0c-bf146921f242", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:53:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "6f417e1d-123a-4bff-8a0c-bf146921f242", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "cb1b1702-d879-4166-b9d8-ebfccd7c1307", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "cb1b1702-d879-4166-b9d8-ebfccd7c1307", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:53:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "cb1b1702-d879-4166-b9d8-ebfccd7c1307", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "85bcf867-f5bc-4222-91b4-1c5a2ea1e0c5", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "85bcf867-f5bc-4222-91b4-1c5a2ea1e0c5", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:53:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "85bcf867-f5bc-4222-91b4-1c5a2ea1e0c5", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:53:52Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "adf311f7-9b11-4a1a-928e-ba679ae3e6c8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:52Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "adf311f7-9b11-4a1a-928e-ba679ae3e6c8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:52Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "adf311f7-9b11-4a1a-928e-ba679ae3e6c8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:53:52Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "adf311f7-9b11-4a1a-928e-ba679ae3e6c8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:53:52Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "adf311f7-9b11-4a1a-928e-ba679ae3e6c8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "authType": "userdefined"} 2026-07-08T18:53:52Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "adf311f7-9b11-4a1a-928e-ba679ae3e6c8", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63"} 2026-07-08T18:54:27Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:54:27Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:54:27Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "e9b8ebc2-d8d7-4c26-b28d-42abdfdf2767", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "12f43640-c3a6-474f-8b0a-76bb0cd6603f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "12f43640-c3a6-474f-8b0a-76bb0cd6603f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:54:27Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "12f43640-c3a6-474f-8b0a-76bb0cd6603f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-2046dd63 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:54:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-trusted-ca-bundle", "reconcileID": "4c19bc1d-e643-4cac-b0ca-df440e037c39", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-trusted-ca-bundle", "reconcileID": "4c19bc1d-e643-4cac-b0ca-df440e037c39", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "openshift-service-ca.crt", "reconcileID": "d44b81dc-e50f-4965-95a3-4acea29e591f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "openshift-service-ca.crt", "reconcileID": "d44b81dc-e50f-4965-95a3-4acea29e591f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1cc6d11-143f-4771-abe1-c3bdb923e436", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1cc6d11-143f-4771-abe1-c3bdb923e436", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "aaa33813-7e5b-45f8-b6b1-1e86b70669b5", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:54:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "aaa33813-7e5b-45f8-b6b1-1e86b70669b5", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:54:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "aaa33813-7e5b-45f8-b6b1-1e86b70669b5", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:54:39Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:39Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63"} 2026-07-08T18:54:39Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:39Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:39Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:54:39Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63-kserve-route-authn"} 2026-07-08T18:54:39Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:39Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "03042b66-62c4-4b39-b828-e7315915eecf", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:39Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-2046dd63","namespace":"ai-tenant-e2e-mt-2046dd63"}, "namespace": "ai-tenant-e2e-mt-2046dd63", "name": "e2e-lifecycle-model-2046dd63", "reconcileID": "6671ab42-8fc1-4fb8-8214-2b73804b770a", "LLMInferenceService": "e2e-lifecycle-model-2046dd63", "namespace": "ai-tenant-e2e-mt-2046dd63"} 2026-07-08T18:54:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "92a72bc4-41bb-4486-8268-942080ce5e4a", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:54:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "92a72bc4-41bb-4486-8268-942080ce5e4a", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63-authn-ssl"} 2026-07-08T18:54:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-2046dd63","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-2046dd63", "reconcileID": "92a72bc4-41bb-4486-8268-942080ce5e4a", "Gateway": "e2e-mt-2046dd63", "namespace": "openshift-ingress"} 2026-07-08T18:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "openshift-service-ca.crt", "reconcileID": "2d30b7c2-f30d-4201-9754-178ea01e3065", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-trusted-ca-bundle", "reconcileID": "03e37b0d-7dab-4b25-b681-84bcb09126a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "openshift-service-ca.crt", "reconcileID": "bfb31e92-737d-4310-a3fe-2093a737ddd9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-0c7092d8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:54:50Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "openshift-service-ca.crt", "reconcileID": "bfb31e92-737d-4310-a3fe-2093a737ddd9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ba307b64-6ffc-43ec-b66c-053717bd267f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "openshift-service-ca.crt", "reconcileID": "d361ecfd-5d31-4f74-803f-5ce8ed98778d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-trusted-ca-bundle", "reconcileID": "ead30a53-8ca3-4028-a508-730ddf4d5b63", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:54:50Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-trusted-ca-bundle", "reconcileID": "ead30a53-8ca3-4028-a508-730ddf4d5b63", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "327f6efa-28a7-4736-970c-94a923cfc559", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "openshift-service-ca.crt", "reconcileID": "8c523060-db81-499e-8b2b-de7e08a00d1e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-trusted-ca-bundle", "reconcileID": "6521017d-d2af-4c10-83e6-a83c38c4ef20", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:51Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "openshift-service-ca.crt", "reconcileID": "b5d4a45e-a3bd-44bf-aac7-d154cd7f2007", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9d707991", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:54:51Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "openshift-service-ca.crt", "reconcileID": "b5d4a45e-a3bd-44bf-aac7-d154cd7f2007", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2d05b3ed-47f7-4948-bd4d-96f39cb2eadb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "openshift-service-ca.crt", "reconcileID": "38396058-7b4b-40cc-8fe0-96cd5ff57a8d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:51Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-trusted-ca-bundle", "reconcileID": "db615ab1-b640-41db-866d-6e0568a1d29d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:54:51Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-trusted-ca-bundle", "reconcileID": "db615ab1-b640-41db-866d-6e0568a1d29d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a548aef-36c4-4c5a-b8cd-14b59e5c5bbd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:54:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bfdab519-8325-461f-b578-8244cf387352", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bfdab519-8325-461f-b578-8244cf387352", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:54:58Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bfdab519-8325-461f-b578-8244cf387352", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-0c7092d8 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:54:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-trusted-ca-bundle", "reconcileID": "14b3c035-395f-4c0d-8232-aa507c525684", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-trusted-ca-bundle", "reconcileID": "14b3c035-395f-4c0d-8232-aa507c525684", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "openshift-service-ca.crt", "reconcileID": "37d1b2ab-e438-4501-a3a3-7136b9580f2b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "openshift-service-ca.crt", "reconcileID": "37d1b2ab-e438-4501-a3a3-7136b9580f2b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4e408e15-be48-4630-9dcf-40be9a237db3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:54:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-0c7092d8"}, "namespace": "ai-tenant-e2e-mt-0c7092d8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4e408e15-be48-4630-9dcf-40be9a237db3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-0c7092d8"} 2026-07-08T18:55:04Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c172a80-dbc2-4868-a752-20fb0f666e73", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:04Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c172a80-dbc2-4868-a752-20fb0f666e73", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:04Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c172a80-dbc2-4868-a752-20fb0f666e73", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-9d707991 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:55:04Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-trusted-ca-bundle", "reconcileID": "fc005378-e4bc-4541-a47c-143ed7958d42", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-trusted-ca-bundle", "reconcileID": "fc005378-e4bc-4541-a47c-143ed7958d42", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:04Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "openshift-service-ca.crt", "reconcileID": "60776a99-ecab-4ad8-8832-42b23dea7cb9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "openshift-service-ca.crt", "reconcileID": "60776a99-ecab-4ad8-8832-42b23dea7cb9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:04Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "68263746-14a8-4635-a1a1-f6dc72da8c0a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9d707991"}, "namespace": "ai-tenant-e2e-mt-9d707991", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "68263746-14a8-4635-a1a1-f6dc72da8c0a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9d707991"} 2026-07-08T18:55:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "642a8fec-3f86-4b7f-8b03-0cb918ee1e25", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-trusted-ca-bundle", "reconcileID": "61ae18e0-2a21-40d0-98a0-9fadbf13b3eb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "42ec6a09-955a-450f-bccb-0f8c5018b6ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "42ec6a09-955a-450f-bccb-0f8c5018b6ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "039bbe5b-6592-471e-b5ff-235922b8479a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:05Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "039bbe5b-6592-471e-b5ff-235922b8479a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:55:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c7a374e-dfae-4b17-8552-270c3ba43870", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "be9bd27e-c3df-429f-b69b-3f46f34c941d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-trusted-ca-bundle", "reconcileID": "a8c5cf83-30c2-4d20-b50f-298695b7c33d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-trusted-ca-bundle", "reconcileID": "a8c5cf83-30c2-4d20-b50f-298695b7c33d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9d69b703-f2c0-4fd2-b021-319671c10314", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "63636a25-0de0-4f1a-9e7b-29c55bd14d2a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "63636a25-0de0-4f1a-9e7b-29c55bd14d2a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:42Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "63636a25-0de0-4f1a-9e7b-29c55bd14d2a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c221d68f because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:55:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-trusted-ca-bundle", "reconcileID": "756ff1c5-0ba2-4243-9660-5c5a4922a74f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-trusted-ca-bundle", "reconcileID": "756ff1c5-0ba2-4243-9660-5c5a4922a74f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "62474e67-7b27-408d-bedc-a09aa34264e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "openshift-service-ca.crt", "reconcileID": "62474e67-7b27-408d-bedc-a09aa34264e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "93f732e9-bdf4-4544-8f98-29788cb7f57b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c221d68f"}, "namespace": "ai-tenant-e2e-mt-c221d68f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "93f732e9-bdf4-4544-8f98-29788cb7f57b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c221d68f"} 2026-07-08T18:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "0b962857-87b4-4eb0-a224-06d0748b3a7c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "0b962857-87b4-4eb0-a224-06d0748b3a7c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "0b962857-87b4-4eb0-a224-06d0748b3a7c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "4017abb0-e252-4c4f-8423-8e1752fe116c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "4017abb0-e252-4c4f-8423-8e1752fe116c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "4017abb0-e252-4c4f-8423-8e1752fe116c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "323b9920-a8d2-4701-968c-de469c609fa9", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "323b9920-a8d2-4701-968c-de469c609fa9", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "323b9920-a8d2-4701-968c-de469c609fa9", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "942d3e75-797b-4584-b615-365e68a5b3e7", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "942d3e75-797b-4584-b615-365e68a5b3e7", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "942d3e75-797b-4584-b615-365e68a5b3e7", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "cecbf46a-7988-4ac8-a093-0aefa684fe5c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "cecbf46a-7988-4ac8-a093-0aefa684fe5c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "cecbf46a-7988-4ac8-a093-0aefa684fe5c", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "3f525a2a-6636-4575-94db-18e8cd1ddb7b", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "3f525a2a-6636-4575-94db-18e8cd1ddb7b", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "3f525a2a-6636-4575-94db-18e8cd1ddb7b", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:44Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "b24cd4f6-ffd2-4820-91c6-c8fd7de7ea30", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:44Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "b24cd4f6-ffd2-4820-91c6-c8fd7de7ea30", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d-authn-ssl"} 2026-07-08T18:55:44Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c2f39d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c2f39d", "reconcileID": "b24cd4f6-ffd2-4820-91c6-c8fd7de7ea30", "Gateway": "e2e-inf-a-c2f39d", "namespace": "openshift-ingress"} 2026-07-08T18:55:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "openshift-service-ca.crt", "reconcileID": "013ec0b9-89d9-492c-bd47-90b0bb8352df", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:55:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "openshift-service-ca.crt", "reconcileID": "0b3d6a24-cba3-473a-9ed5-5054c594a2ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c2f39d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:46Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "openshift-service-ca.crt", "reconcileID": "0b3d6a24-cba3-473a-9ed5-5054c594a2ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:55:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-trusted-ca-bundle", "reconcileID": "d7116607-e85c-484f-a83f-9ef325e33781", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:55:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "openshift-service-ca.crt", "reconcileID": "5034b85d-eb21-4ea5-a854-19e2a1ad2c4a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:55:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c67070a8-9e87-4955-a6a9-0010b31426eb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:55:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-trusted-ca-bundle", "reconcileID": "6b953441-2d62-4ccd-8b0d-f16ad176a224", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:55:46Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-trusted-ca-bundle", "reconcileID": "6b953441-2d62-4ccd-8b0d-f16ad176a224", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:55:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "05b3f404-676c-4f16-a933-5ec1d54e5bc7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:56:55Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T18:56:55Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T18:56:57Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T18:56:57Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T18:59:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2db72e94-4137-42f3-a904-1a3c275d4ac0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:59:01Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2db72e94-4137-42f3-a904-1a3c275d4ac0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T18:59:01Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2db72e94-4137-42f3-a904-1a3c275d4ac0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-c2f39d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T18:59:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-trusted-ca-bundle", "reconcileID": "c7dbd016-2bed-43bb-b3ed-218483acdd36", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:59:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-trusted-ca-bundle", "reconcileID": "c7dbd016-2bed-43bb-b3ed-218483acdd36", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:59:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "openshift-service-ca.crt", "reconcileID": "0f165fea-3dd1-471d-842d-9fbf144d2321", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:59:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "openshift-service-ca.crt", "reconcileID": "0f165fea-3dd1-471d-842d-9fbf144d2321", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:59:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d4140003-6b45-42e2-a043-fc2a4c052f1d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"} 2026-07-08T18:59:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c2f39d"}, "namespace": "ai-tenant-e2e-inf-a-c2f39d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d4140003-6b45-42e2-a043-fc2a4c052f1d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c2f39d"}