2026-06-11T20:28:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "de9b088f-da2c-407c-9cd4-d7011607d844", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T20:28:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "de9b088f-da2c-407c-9cd4-d7011607d844", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T20:28:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d50afd0c-18d1-4af4-a323-40a367e0d3a9", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T20:28:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d50afd0c-18d1-4af4-a323-40a367e0d3a9", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T20:28:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d50afd0c-18d1-4af4-a323-40a367e0d3a9", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T20:29:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b0a7da56-7c75-467e-a3a1-2520eda9f2b5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T20:29:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b0a7da56-7c75-467e-a3a1-2520eda9f2b5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T20:29:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b0a7da56-7c75-467e-a3a1-2520eda9f2b5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T20:29:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "9a491ee3-accc-433c-a603-38a2d9486c30", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "85b29cba-ff76-46dd-9323-44b194982730", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "308263b3-01ae-43f8-8ffb-f87f92091bac", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "2156921b-221e-4f99-bc17-2d6a7f8f5fab", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "69747352-307f-48c3-9273-4aad7476b631", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "openshift-service-ca.crt", "reconcileID": "284a6efd-4950-4e0e-9d2d-65ddd024bbaf", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-trusted-ca-bundle", "reconcileID": "db8765d9-cc1c-4856-b0cd-3f5d0f867568", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-68645d30", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-trusted-ca-bundle", "reconcileID": "db8765d9-cc1c-4856-b0cd-3f5d0f867568", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "openshift-service-ca.crt", "reconcileID": "993ce193-3bde-45ae-93fb-5aebe0cb8716", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-68645d30", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:05Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "openshift-service-ca.crt", "reconcileID": "993ce193-3bde-45ae-93fb-5aebe0cb8716", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:29:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "becedcad-40d1-4bcb-a22a-36b39cf71085", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "openshift-service-ca.crt", "reconcileID": "b604c089-61ef-4a28-9af9-7c21ee0c3784", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-trusted-ca-bundle", "reconcileID": "f8414535-2232-4425-bcd7-565aabee4565", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-68645d30", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-trusted-ca-bundle", "reconcileID": "f8414535-2232-4425-bcd7-565aabee4565", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c000d8ed-3038-4fd9-be6b-fb2042bbbf60", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "08e1f87b-7538-4f4c-ab0c-29cb4bc8c84a", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:09Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-68645d30","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-68645d30", "reconcileID": "606be27c-794d-41c2-ad37-a00d80b54695", "Gateway": "e2e-ait-68645d30", "namespace": "openshift-ingress"} 2026-06-11T20:29:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7ed794da-d2a3-4843-a6b6-f29df1f240ef", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7ed794da-d2a3-4843-a6b6-f29df1f240ef", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-68645d30", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:17Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7ed794da-d2a3-4843-a6b6-f29df1f240ef", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-68645d30 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:29:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-trusted-ca-bundle", "reconcileID": "0420600e-9421-4098-a3c0-3839301039d9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-trusted-ca-bundle", "reconcileID": "0420600e-9421-4098-a3c0-3839301039d9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "openshift-service-ca.crt", "reconcileID": "c0f438bc-f3a0-4e7e-9480-d3facd739d1e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "openshift-service-ca.crt", "reconcileID": "c0f438bc-f3a0-4e7e-9480-d3facd739d1e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "204902bb-5dfb-43c7-9aa8-da82c54c52df", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-68645d30"}, "namespace": "e2e-aitenant-68645d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "204902bb-5dfb-43c7-9aa8-da82c54c52df", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-68645d30"} 2026-06-11T20:29:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "028352f8-5b1a-4172-b94f-440c2a81eabc", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "a7ad9afd-1c40-43ca-a196-9061b103a780", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "e22635c6-c5be-4660-b005-fe4f4a2b6612", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "fdda3151-8e62-4371-8a82-10576ee7af21", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "a85a239e-059e-4a88-8fa3-0bc65b6d97b1", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "openshift-service-ca.crt", "reconcileID": "e5a34ecc-d571-413c-bd7a-e96247d146f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-trusted-ca-bundle", "reconcileID": "2b8c7525-39ba-4625-a0d4-cc079e0e40fb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "openshift-service-ca.crt", "reconcileID": "112a821d-9623-4bc0-a04e-f3f177e13275", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-b3e5ed43", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "openshift-service-ca.crt", "reconcileID": "112a821d-9623-4bc0-a04e-f3f177e13275", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c2b9a6e-6eea-41d9-b0dc-6ca780d6f381", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "openshift-service-ca.crt", "reconcileID": "d0b449c6-598f-40b2-bf06-68caabd75664", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-trusted-ca-bundle", "reconcileID": "211cb8e0-7d9e-49bc-a4d8-1fa848116fe0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-trusted-ca-bundle", "reconcileID": "211cb8e0-7d9e-49bc-a4d8-1fa848116fe0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "84afad4d-4a8e-47ec-9af7-a3a86cdc50ad", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:19Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "c5075e57-3c28-4745-b880-436b1ad98077", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-b3e5ed43","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-b3e5ed43", "reconcileID": "6b35c2be-499f-4eed-9fea-10eade480291", "Gateway": "e2e-ait-b3e5ed43", "namespace": "openshift-ingress"} 2026-06-11T20:29:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3af74be4-49d7-4fa0-b4e2-613a0e800fe5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3af74be4-49d7-4fa0-b4e2-613a0e800fe5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3af74be4-49d7-4fa0-b4e2-613a0e800fe5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-b3e5ed43 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:29:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-trusted-ca-bundle", "reconcileID": "98163cc1-ded3-408e-83c1-d4ec923de135", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-trusted-ca-bundle", "reconcileID": "98163cc1-ded3-408e-83c1-d4ec923de135", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "openshift-service-ca.crt", "reconcileID": "e7e0244b-0bfd-4bba-b573-fc33ea41cc28", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "openshift-service-ca.crt", "reconcileID": "e7e0244b-0bfd-4bba-b573-fc33ea41cc28", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c25a1746-9a27-4045-be25-5d66d8a56d3e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-b3e5ed43"}, "namespace": "e2e-aitenant-b3e5ed43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c25a1746-9a27-4045-be25-5d66d8a56d3e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-b3e5ed43"} 2026-06-11T20:29:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-d8cf27af","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-d8cf27af", "reconcileID": "5d8a0eff-94bf-4900-8f44-804b961a0816", "Gateway": "e2e-invalid-d8cf27af", "namespace": "openshift-ingress"} 2026-06-11T20:29:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-d8cf27af","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-d8cf27af", "reconcileID": "c3883540-492c-4733-837d-cb32a2a9f233", "Gateway": "e2e-invalid-d8cf27af", "namespace": "openshift-ingress"} 2026-06-11T20:29:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-d8cf27af","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-d8cf27af", "reconcileID": "0b94f677-d9e9-44a4-b2cf-768d283efda3", "Gateway": "e2e-invalid-d8cf27af", "namespace": "openshift-ingress"} 2026-06-11T20:29:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-d8cf27af","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-d8cf27af", "reconcileID": "5a6d12d3-6a1e-495e-ab78-9d61a83c4ada", "Gateway": "e2e-invalid-d8cf27af", "namespace": "openshift-ingress"} 2026-06-11T20:29:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-d8cf27af","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-d8cf27af", "reconcileID": "63d4cceb-7c39-4c15-905a-38abe543dab7", "Gateway": "e2e-invalid-d8cf27af", "namespace": "openshift-ingress"} 2026-06-11T20:29:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "openshift-service-ca.crt", "reconcileID": "0240e2eb-8e9d-4da6-ba52-2acfa204a83a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-trusted-ca-bundle", "reconcileID": "4d6bbb97-1683-40b6-9a2b-3dd3dd47aed9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "openshift-service-ca.crt", "reconcileID": "d3949c43-fec4-4621-a9b7-873ad45ab4d4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-72bef175", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "openshift-service-ca.crt", "reconcileID": "d3949c43-fec4-4621-a9b7-873ad45ab4d4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59271aac-8914-4f43-b363-10cc91288176", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "openshift-service-ca.crt", "reconcileID": "c78a6588-d9ca-4bd9-af76-b9951094b58d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-trusted-ca-bundle", "reconcileID": "09f7ec44-4e9b-42e2-a329-a61606fb6d2f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-72bef175", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-trusted-ca-bundle", "reconcileID": "09f7ec44-4e9b-42e2-a329-a61606fb6d2f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8a640de1-fd05-462c-a9c6-2473584e4d55", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d34a41cd-965f-4524-ae0d-b19366bbf46f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d34a41cd-965f-4524-ae0d-b19366bbf46f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-72bef175", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:39Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d34a41cd-965f-4524-ae0d-b19366bbf46f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-72bef175 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:29:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-trusted-ca-bundle", "reconcileID": "1651c316-808b-420f-bc28-3aa1fe2b7e89", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-trusted-ca-bundle", "reconcileID": "1651c316-808b-420f-bc28-3aa1fe2b7e89", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "openshift-service-ca.crt", "reconcileID": "b66abb0e-2a67-4bdf-89ae-280dc657b7f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "openshift-service-ca.crt", "reconcileID": "b66abb0e-2a67-4bdf-89ae-280dc657b7f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9fee93ef-1968-4f93-9330-73b7c62bb97c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-72bef175"}, "namespace": "e2e-mt-72bef175", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9fee93ef-1968-4f93-9330-73b7c62bb97c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-72bef175"} 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "openshift-service-ca.crt", "reconcileID": "51eec1e6-92dc-40a5-be79-853e56705aa7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:29:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "openshift-service-ca.crt", "reconcileID": "bab2d0f4-1e78-4206-b052-9d654050d594", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-60ae6326", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:39Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "openshift-service-ca.crt", "reconcileID": "bab2d0f4-1e78-4206-b052-9d654050d594", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:29:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "2f463294-2565-4e22-a3a9-e53553bcbe17", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-60ae6326", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:39Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "2f463294-2565-4e22-a3a9-e53553bcbe17", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "openshift-service-ca.crt", "reconcileID": "db150eac-46ce-4c14-9a6b-50e2d04d212f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "78711311-56e9-4e0c-a7d5-5e4d9df69c32", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:29:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "530cb0bf-1606-4d29-a10b-bf04d63495c7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:29:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "1eed09b7-51fe-4cef-bd80-910b28053227", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-60ae6326", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:29:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "1eed09b7-51fe-4cef-bd80-910b28053227", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:29:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b0f0643-7d9a-491c-9afd-8a4cb1c48ea8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e3a60fb-eb05-468e-abcd-6022df02bac2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e3a60fb-eb05-468e-abcd-6022df02bac2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-60ae6326", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:12Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e3a60fb-eb05-468e-abcd-6022df02bac2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-60ae6326 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:30:12Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "cce465d8-9701-4348-ae62-035fee4e4a68", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-trusted-ca-bundle", "reconcileID": "cce465d8-9701-4348-ae62-035fee4e4a68", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "openshift-service-ca.crt", "reconcileID": "03ecffc5-35b3-433e-a942-041ee6b0868e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "openshift-service-ca.crt", "reconcileID": "03ecffc5-35b3-433e-a942-041ee6b0868e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eb5b64cb-4d39-466f-bab7-a400346dfe12", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-60ae6326"}, "namespace": "e2e-mt-60ae6326", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eb5b64cb-4d39-466f-bab7-a400346dfe12", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-60ae6326"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "openshift-service-ca.crt", "reconcileID": "9b164f82-cc15-4045-ab9d-3b24b1a1ff2a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "openshift-service-ca.crt", "reconcileID": "2bf62a2c-4797-43e2-b421-c6c920723d8e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-af6ec450", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "openshift-service-ca.crt", "reconcileID": "2bf62a2c-4797-43e2-b421-c6c920723d8e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-trusted-ca-bundle", "reconcileID": "a9e5ace9-e5c1-4787-9a9a-8f4740c17c2e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "21c5abc5-d704-4053-a93d-4bbc5ea80d36", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "openshift-service-ca.crt", "reconcileID": "7fb93a94-6a76-43d5-b667-81a8739f8d08", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-trusted-ca-bundle", "reconcileID": "5c0d3ddb-19b1-4c09-be39-82e2ea96a372", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-af6ec450", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-trusted-ca-bundle", "reconcileID": "5c0d3ddb-19b1-4c09-be39-82e2ea96a372", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f6c60b6-b05f-49a0-b1c7-510ea1c1f3b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c2b2d97-a2b9-4e4b-9dc2-4dde2a3f659b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c2b2d97-a2b9-4e4b-9dc2-4dde2a3f659b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-af6ec450", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:33Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c2b2d97-a2b9-4e4b-9dc2-4dde2a3f659b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-af6ec450 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:30:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-trusted-ca-bundle", "reconcileID": "51963122-056b-4a9b-998e-f0805d4dab52", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-trusted-ca-bundle", "reconcileID": "51963122-056b-4a9b-998e-f0805d4dab52", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "openshift-service-ca.crt", "reconcileID": "aa4bce77-6c00-4ba7-a89b-48cd7b98f5e6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "openshift-service-ca.crt", "reconcileID": "aa4bce77-6c00-4ba7-a89b-48cd7b98f5e6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f2ab3cb-903a-48d9-84a5-7b3b92fabd38", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-af6ec450"}, "namespace": "e2e-unlabeled-af6ec450", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f2ab3cb-903a-48d9-84a5-7b3b92fabd38", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-af6ec450"} 2026-06-11T20:30:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "openshift-service-ca.crt", "reconcileID": "6169a500-a2e2-4ac8-81e2-722db87df844", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-trusted-ca-bundle", "reconcileID": "35ec1e85-13f4-4316-89d1-cb89789db329", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "openshift-service-ca.crt", "reconcileID": "4a0f9feb-19e7-4f7b-a08b-c2c8c600173a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ad76cd3a", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:34Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "openshift-service-ca.crt", "reconcileID": "4a0f9feb-19e7-4f7b-a08b-c2c8c600173a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "845e8777-63e5-47cf-a044-f3d94187f3e5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "openshift-service-ca.crt", "reconcileID": "2a2bdb0a-cb17-41f6-ac9f-74e474986ae4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-trusted-ca-bundle", "reconcileID": "1399fb1a-6c9a-4fc9-8f5b-cb404110aa34", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ad76cd3a", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:34Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-trusted-ca-bundle", "reconcileID": "1399fb1a-6c9a-4fc9-8f5b-cb404110aa34", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1d3add6c-d919-402e-b4ba-6a6612d8dff6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bf4411d4-2dfe-459a-8e0f-5a85ab653ac0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bf4411d4-2dfe-459a-8e0f-5a85ab653ac0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ad76cd3a", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:51Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bf4411d4-2dfe-459a-8e0f-5a85ab653ac0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-ad76cd3a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:30:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-trusted-ca-bundle", "reconcileID": "9f48a0e2-f49c-4d05-9889-718b268e8aaa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-trusted-ca-bundle", "reconcileID": "9f48a0e2-f49c-4d05-9889-718b268e8aaa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "openshift-service-ca.crt", "reconcileID": "3a114eb1-9d13-4acb-bd26-2dbac825a77b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "openshift-service-ca.crt", "reconcileID": "3a114eb1-9d13-4acb-bd26-2dbac825a77b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c3555faa-49b6-4a02-8a7c-cd0690db802a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ad76cd3a"}, "namespace": "e2e-mt-ad76cd3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c3555faa-49b6-4a02-8a7c-cd0690db802a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ad76cd3a"} 2026-06-11T20:30:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "openshift-service-ca.crt", "reconcileID": "cefec9a2-cd8d-4850-acf2-d6566aa716b9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-trusted-ca-bundle", "reconcileID": "30032c12-39c5-4ef3-a1bf-351dd6572983", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:52Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "openshift-service-ca.crt", "reconcileID": "59836e39-ece2-445a-b2d5-6bc32f3d770d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-26d5ce87", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:52Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "openshift-service-ca.crt", "reconcileID": "59836e39-ece2-445a-b2d5-6bc32f3d770d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "26263e0a-e29e-41b0-b185-d11e8dd2cb8d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "openshift-service-ca.crt", "reconcileID": "67270501-8150-4296-b279-127595fb3041", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:52Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-trusted-ca-bundle", "reconcileID": "82879218-066d-4cd9-82f1-c4af55515cdd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-26d5ce87", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:52Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-trusted-ca-bundle", "reconcileID": "82879218-066d-4cd9-82f1-c4af55515cdd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "42fb11da-db35-496b-bd77-2d95b7e27f58", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ac162bed-29a1-467d-88b0-687c54d7374c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ac162bed-29a1-467d-88b0-687c54d7374c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-26d5ce87", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:58Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ac162bed-29a1-467d-88b0-687c54d7374c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-26d5ce87 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:30:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-trusted-ca-bundle", "reconcileID": "b128145e-2ce1-4257-bd5a-23af44221e53", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-trusted-ca-bundle", "reconcileID": "b128145e-2ce1-4257-bd5a-23af44221e53", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "openshift-service-ca.crt", "reconcileID": "9d13b858-e97b-4fd3-bc5e-56904c183ffc", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "openshift-service-ca.crt", "reconcileID": "9d13b858-e97b-4fd3-bc5e-56904c183ffc", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b814fc3f-ffe6-4dc4-92b2-a597d564b46a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-26d5ce87"}, "namespace": "e2e-mt-26d5ce87", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b814fc3f-ffe6-4dc4-92b2-a597d564b46a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-26d5ce87"} 2026-06-11T20:30:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "openshift-service-ca.crt", "reconcileID": "82ef4eba-a418-4c83-860d-8b2da41469ac", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:30:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-trusted-ca-bundle", "reconcileID": "c53bd7aa-e275-4ddc-9732-6783c6aa4650", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:30:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "openshift-service-ca.crt", "reconcileID": "73377eea-a5d9-4ae0-ad04-509844d5910c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-61f12692", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "openshift-service-ca.crt", "reconcileID": "73377eea-a5d9-4ae0-ad04-509844d5910c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:30:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "openshift-service-ca.crt", "reconcileID": "1004000f-5e0b-474a-9572-55f89864b950", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:30:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f47c173-67a9-4caf-a77a-48a750b71679", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:30:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-trusted-ca-bundle", "reconcileID": "6b42a9db-c331-4896-bc35-ca625c17e7f7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-61f12692", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:30:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-trusted-ca-bundle", "reconcileID": "6b42a9db-c331-4896-bc35-ca625c17e7f7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:30:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a885648-8f2f-488f-8f47-ca38412d479a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "openshift-service-ca.crt", "reconcileID": "a219ed95-5861-4f91-8910-139a3e7e9be1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-trusted-ca-bundle", "reconcileID": "f61c903e-16d9-460f-9819-a8ec71ec7551", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "openshift-service-ca.crt", "reconcileID": "1dadbf24-76f8-49c2-a71b-ef0224718446", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-cdca84da", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "openshift-service-ca.crt", "reconcileID": "1dadbf24-76f8-49c2-a71b-ef0224718446", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2d87a0ab-fa10-4beb-b6d0-3ce0dd27283a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "openshift-service-ca.crt", "reconcileID": "0cf5cb6b-0b38-430e-a59f-c745e3634b87", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-trusted-ca-bundle", "reconcileID": "ac670df5-0bc3-4094-8f1d-e70f235d55b3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-cdca84da", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-trusted-ca-bundle", "reconcileID": "ac670df5-0bc3-4094-8f1d-e70f235d55b3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1aa81a69-1963-4726-9764-4db41f751427", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b189c93-0f36-4281-8232-a0c376fadc8a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b189c93-0f36-4281-8232-a0c376fadc8a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-61f12692", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:07Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b189c93-0f36-4281-8232-a0c376fadc8a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-61f12692 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-trusted-ca-bundle", "reconcileID": "585e357f-a97c-4c77-ad5f-63e8fa3345f2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-trusted-ca-bundle", "reconcileID": "585e357f-a97c-4c77-ad5f-63e8fa3345f2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "openshift-service-ca.crt", "reconcileID": "1c78b431-f443-4aca-8a3a-58369af98a81", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "openshift-service-ca.crt", "reconcileID": "1c78b431-f443-4aca-8a3a-58369af98a81", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6049c1ad-1480-4bc5-8f43-f0cc31c9da7b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-61f12692"}, "namespace": "e2e-mt-61f12692", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6049c1ad-1480-4bc5-8f43-f0cc31c9da7b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-61f12692"} 2026-06-11T20:31:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e2425fa-3b9e-42e1-89a1-b90b1ff5e2ed", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e2425fa-3b9e-42e1-89a1-b90b1ff5e2ed", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-cdca84da", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e2425fa-3b9e-42e1-89a1-b90b1ff5e2ed", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-cdca84da because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-trusted-ca-bundle", "reconcileID": "12bc72ec-12fd-4a41-95f0-0d774e5021e1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-trusted-ca-bundle", "reconcileID": "12bc72ec-12fd-4a41-95f0-0d774e5021e1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "openshift-service-ca.crt", "reconcileID": "55e15ef9-cb1f-40ec-9298-6fd8084ec118", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "openshift-service-ca.crt", "reconcileID": "55e15ef9-cb1f-40ec-9298-6fd8084ec118", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f018f446-8203-4f6c-9f57-8a9c156c4286", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-cdca84da"}, "namespace": "e2e-mt-cdca84da", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f018f446-8203-4f6c-9f57-8a9c156c4286", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-cdca84da"} 2026-06-11T20:31:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "9ce8e1e4-1d69-44e5-8e83-8442f2fe7196", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "70b35b09-3702-45af-a7d5-205bc5763f99", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "0a445b63-a47e-4098-abb7-ba1aded74c11", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "3ff96874-3e2b-45fd-9523-00d7764d8e90", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "314476ff-d003-4927-aea2-39e727e9eae8", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "6a3d098f-8fc6-4277-a53c-e1fe6672112e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "d28de49f-c4af-4e61-9ce4-d5a06e81ac72", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "9c0caa7a-4f00-4885-bbc6-db0ffd706bde", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:15Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "9c0caa7a-4f00-4885-bbc6-db0ffd706bde", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "71c29770-135f-4c62-a154-d969a7ed8926", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:15Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "71c29770-135f-4c62-a154-d969a7ed8926", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "66317cb4-f8f2-4c1c-a707-d4c41286f542", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "7bbb7bf2-a5dd-49c3-9d75-605778606d16", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "17a2231f-c78b-4910-9be4-86c51f9bbf7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-6b7655d5", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:15Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "17a2231f-c78b-4910-9be4-86c51f9bbf7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "028af743-048d-4d73-a6e9-fc49badfb00c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "ba42e5cc-b602-4947-8be6-0aa272b2d370", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-6b7655d5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-6b7655d5", "reconcileID": "bbfa4a9f-94e4-4cc8-8c0f-364d470ae19e", "Gateway": "e2e-mt-6b7655d5", "namespace": "openshift-ingress"} 2026-06-11T20:31:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "bbdf1e50-7397-413e-856e-75b57d319177", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "703eb2d3-8df7-4727-8802-703f026eaa70", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "2887cf50-d1bf-4bb2-a181-ff48bab010db", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-6b7655d5", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:20Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "2887cf50-d1bf-4bb2-a181-ff48bab010db", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "2c80c318-aee2-40db-9534-3d2caa18609d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "65ef281f-0209-4e14-b1f7-a61cfdb64a52", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "edef22f1-ea5a-4ff2-86cd-a7212ed90b02", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "edef22f1-ea5a-4ff2-86cd-a7212ed90b02", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b39441e4-65f5-465e-8ad5-d54c9eb5b7c7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fc97032b-ed4f-4d58-b338-fb1299aa466f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fc97032b-ed4f-4d58-b338-fb1299aa466f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-6b7655d5", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:26Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fc97032b-ed4f-4d58-b338-fb1299aa466f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-other-6b7655d5 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "798e1af5-4a6e-4762-a359-6be507f53c3b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "798e1af5-4a6e-4762-a359-6be507f53c3b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:26Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "798e1af5-4a6e-4762-a359-6be507f53c3b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-other-6b7655d5 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "556027a4-8eef-4c68-a3d0-b567d6ba43d4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "556027a4-8eef-4c68-a3d0-b567d6ba43d4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f975e213-8ed8-48b4-ad25-33c2034aeede", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f975e213-8ed8-48b4-ad25-33c2034aeede", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "7e30f376-f1bf-48a5-8352-b84dd4dcba26", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-6b7655d5"}, "namespace": "e2e-mt-other-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "7e30f376-f1bf-48a5-8352-b84dd4dcba26", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-6b7655d5"} 2026-06-11T20:31:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9444fc94-f58e-4273-9e6e-7d41bca23be9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9444fc94-f58e-4273-9e6e-7d41bca23be9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-6b7655d5", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:33Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9444fc94-f58e-4273-9e6e-7d41bca23be9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-6b7655d5 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "f1321afc-1bb8-4e87-8345-b8f0a3c5ab2a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-trusted-ca-bundle", "reconcileID": "f1321afc-1bb8-4e87-8345-b8f0a3c5ab2a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "ce186c48-b17b-4497-9efd-956966288599", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "openshift-service-ca.crt", "reconcileID": "ce186c48-b17b-4497-9efd-956966288599", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "458b244f-c58c-415e-9f73-50069ff51807", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-6b7655d5"}, "namespace": "e2e-mt-6b7655d5", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "458b244f-c58c-415e-9f73-50069ff51807", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-6b7655d5"} 2026-06-11T20:31:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "openshift-service-ca.crt", "reconcileID": "2d73b4c2-4afd-4fea-ba01-f29487e99fe1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-trusted-ca-bundle", "reconcileID": "16eed268-c9f9-47b0-a924-48c66ba42398", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "openshift-service-ca.crt", "reconcileID": "1a617c92-ca7a-4799-9e33-8044776c057a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8fec9fb7", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:34Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "openshift-service-ca.crt", "reconcileID": "1a617c92-ca7a-4799-9e33-8044776c057a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c3a0c1fa-180e-41c8-bea1-f5edaf7c768e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "openshift-service-ca.crt", "reconcileID": "8a13ac49-37b9-462c-8bc0-58bc6c8908b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-trusted-ca-bundle", "reconcileID": "bc6a8498-967c-4c82-aba9-2f55173f4c8c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:34Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-trusted-ca-bundle", "reconcileID": "bc6a8498-967c-4c82-aba9-2f55173f4c8c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74b52096-b109-468f-8421-a324e4f9da2e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6638fdd7-358c-4996-93b8-1f36018f7f50", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6638fdd7-358c-4996-93b8-1f36018f7f50", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:40Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6638fdd7-358c-4996-93b8-1f36018f7f50", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-8fec9fb7 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-trusted-ca-bundle", "reconcileID": "22a52905-8c4d-4b0f-a862-a23b69dd90f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-trusted-ca-bundle", "reconcileID": "22a52905-8c4d-4b0f-a862-a23b69dd90f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "openshift-service-ca.crt", "reconcileID": "5f7a9868-fe4e-46a1-9d6a-331d263e56ec", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "openshift-service-ca.crt", "reconcileID": "5f7a9868-fe4e-46a1-9d6a-331d263e56ec", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b3c99fd-ff4a-42c9-845a-08cfd933cccc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8fec9fb7"}, "namespace": "e2e-webhook-sub-8fec9fb7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b3c99fd-ff4a-42c9-845a-08cfd933cccc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8fec9fb7"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "87e331b8-8d88-4533-a918-0813f948b6f1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-trusted-ca-bundle", "reconcileID": "d6cfd6da-3f2d-41fc-9ea0-b61d51dabee4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "1c665abc-54b1-49e8-afdf-afc047bf22e9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "1c665abc-54b1-49e8-afdf-afc047bf22e9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "ed58c467-d6aa-4c3b-92f1-ca848c05d8d3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:40Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "ed58c467-d6aa-4c3b-92f1-ca848c05d8d3", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1e562f87-5f0c-410e-99bf-a63f87237422", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "dc213e49-d87d-4f36-8c2d-40abcdbe48e2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-trusted-ca-bundle", "reconcileID": "f5397626-bbb2-453d-96eb-8f5f3d4f160b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-trusted-ca-bundle", "reconcileID": "f5397626-bbb2-453d-96eb-8f5f3d4f160b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8ba405e1-8f2d-4766-8eaa-ec2ab273561b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "df7171ab-874c-4982-b191-0c3c66ff1cf2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "df7171ab-874c-4982-b191-0c3c66ff1cf2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:46Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "df7171ab-874c-4982-b191-0c3c66ff1cf2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-7f6044fd because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:31:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-trusted-ca-bundle", "reconcileID": "bb77bb58-b975-4188-a13f-d6acc720a241", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-trusted-ca-bundle", "reconcileID": "bb77bb58-b975-4188-a13f-d6acc720a241", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "b67ada3a-f4eb-4ab5-a985-5d99fde98285", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "openshift-service-ca.crt", "reconcileID": "b67ada3a-f4eb-4ab5-a985-5d99fde98285", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ade05bd-b220-4515-a916-84eae1e1a9cb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-7f6044fd"}, "namespace": "e2e-webhook-auth-7f6044fd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ade05bd-b220-4515-a916-84eae1e1a9cb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-7f6044fd"} 2026-06-11T20:31:55Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T20:31:55Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T20:31:57Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T20:31:57Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T20:31:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "6ded5d9a-c6e1-462d-936a-c9d5a56923b4", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:31:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "4b0b398c-5ca9-4443-9f2b-80b876dfb69d", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:31:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "b46a77ad-bb0e-489b-bc68-36e99e9ba2f5", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:31:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "1e4942a3-0f55-4c5e-a527-cc552a47835d", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:31:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "31f68db3-a822-4adf-9e26-d8ee3629b4ee", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:31:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-trusted-ca-bundle", "reconcileID": "68437ab8-af6b-4d6a-9099-b78112b5cfd5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "openshift-service-ca.crt", "reconcileID": "2d156228-0704-448c-9a79-2bb466aea06e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "openshift-service-ca.crt", "reconcileID": "fe035b0a-94da-4a09-ae58-938545bbe46f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-fc9a85bf", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "openshift-service-ca.crt", "reconcileID": "fe035b0a-94da-4a09-ae58-938545bbe46f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b6ab4987-5dc3-4cc5-a140-1ab5495d5317", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "openshift-service-ca.crt", "reconcileID": "f757b734-691f-47e5-bd1f-4dbbe2381fef", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-trusted-ca-bundle", "reconcileID": "b7ed7d97-e69d-4002-86b8-06f54efb0c5f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-fc9a85bf", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:31:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-trusted-ca-bundle", "reconcileID": "b7ed7d97-e69d-4002-86b8-06f54efb0c5f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "08f28dc8-5778-4288-8c29-8562b5ec4bc5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:31:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "8c3e0fe3-e775-4be5-8dd2-12d98ba723b2", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:32:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-fc9a85bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-fc9a85bf", "reconcileID": "e1181e1f-8d22-4de6-b1ed-72b3a55223e0", "Gateway": "e2e-mt-fc9a85bf", "namespace": "openshift-ingress"} 2026-06-11T20:34:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "165cca94-abdf-4206-9d35-486fc8a5ba35", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "165cca94-abdf-4206-9d35-486fc8a5ba35", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-fc9a85bf", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "165cca94-abdf-4206-9d35-486fc8a5ba35", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-fc9a85bf because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:34:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-trusted-ca-bundle", "reconcileID": "66f29e93-3e75-42c8-869f-c4f1ee47100f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-trusted-ca-bundle", "reconcileID": "66f29e93-3e75-42c8-869f-c4f1ee47100f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "openshift-service-ca.crt", "reconcileID": "20029145-115a-454f-9c71-1d68ea291ced", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "openshift-service-ca.crt", "reconcileID": "20029145-115a-454f-9c71-1d68ea291ced", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1a11325b-621a-404f-b594-f9424e55a48a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-fc9a85bf"}, "namespace": "e2e-mt-fc9a85bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1a11325b-621a-404f-b594-f9424e55a48a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-fc9a85bf"} 2026-06-11T20:34:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "openshift-service-ca.crt", "reconcileID": "fca7fa6a-17f3-4d62-bad9-848ffde9f60b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-trusted-ca-bundle", "reconcileID": "fb3aa89c-c8fe-4299-a093-b02d24b18ecf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-c79ec744", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:15Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-trusted-ca-bundle", "reconcileID": "fb3aa89c-c8fe-4299-a093-b02d24b18ecf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "openshift-service-ca.crt", "reconcileID": "7f1dd076-9c16-4f83-9163-8d793c8e6569", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-c79ec744", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:15Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "openshift-service-ca.crt", "reconcileID": "7f1dd076-9c16-4f83-9163-8d793c8e6569", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:34:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4d96a7a8-75a3-46ee-b26c-28408e0083b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "openshift-service-ca.crt", "reconcileID": "190a4fa9-def8-42b1-b38d-39aa4aceb66a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-trusted-ca-bundle", "reconcileID": "aea06d8f-573e-486c-accb-8542c8058aec", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-c79ec744", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:15Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-trusted-ca-bundle", "reconcileID": "aea06d8f-573e-486c-accb-8542c8058aec", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "868aaa6f-f2f7-4f56-a303-47cf6916bfe6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "openshift-service-ca.crt", "reconcileID": "bfeb0afd-d6f1-40cd-9b79-8ca057c2ecdd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-trusted-ca-bundle", "reconcileID": "29d09c22-4f85-4ceb-9c2c-a951474e1f3a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "openshift-service-ca.crt", "reconcileID": "6f70d340-06fd-4100-9043-4a7979b483f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-3bbeb63d", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:16Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "openshift-service-ca.crt", "reconcileID": "6f70d340-06fd-4100-9043-4a7979b483f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bd9cdf05-afec-45ea-b114-27ac057a3281", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "openshift-service-ca.crt", "reconcileID": "95c8451c-ac7d-456d-8c09-413bdbbfe6f7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-trusted-ca-bundle", "reconcileID": "c9885d34-d305-47a8-b693-3bd2816bfff9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-3bbeb63d", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:16Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-trusted-ca-bundle", "reconcileID": "c9885d34-d305-47a8-b693-3bd2816bfff9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e0818ea9-6546-4a69-8960-74a430e2aa10", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6aed78e7-7e5e-45ea-bd8a-0a5033b741d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:23Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6aed78e7-7e5e-45ea-bd8a-0a5033b741d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-c79ec744", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:23Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6aed78e7-7e5e-45ea-bd8a-0a5033b741d6", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-c79ec744 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:34:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-trusted-ca-bundle", "reconcileID": "f0a50b1b-a4cf-490c-b03f-af668d18c4f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-trusted-ca-bundle", "reconcileID": "f0a50b1b-a4cf-490c-b03f-af668d18c4f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "openshift-service-ca.crt", "reconcileID": "94700135-1089-4668-8a4e-610c436d9816", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "openshift-service-ca.crt", "reconcileID": "94700135-1089-4668-8a4e-610c436d9816", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f4dbf261-f2ed-4fc4-8f1e-a2b1ba236169", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-c79ec744"}, "namespace": "e2e-mt-c79ec744", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f4dbf261-f2ed-4fc4-8f1e-a2b1ba236169", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-c79ec744"} 2026-06-11T20:34:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a71578b-909c-44ff-8ae7-aaf165ff48a5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a71578b-909c-44ff-8ae7-aaf165ff48a5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-3bbeb63d", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a71578b-909c-44ff-8ae7-aaf165ff48a5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-3bbeb63d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:34:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-trusted-ca-bundle", "reconcileID": "c879c8bb-018c-47d3-9d99-2210dc64e71a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-trusted-ca-bundle", "reconcileID": "c879c8bb-018c-47d3-9d99-2210dc64e71a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "openshift-service-ca.crt", "reconcileID": "bbc91699-5285-448a-85d4-824ded5d0423", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "openshift-service-ca.crt", "reconcileID": "bbc91699-5285-448a-85d4-824ded5d0423", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0e84574b-f7fe-45e9-bfe7-b405e3cba17e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-3bbeb63d"}, "namespace": "e2e-mt-3bbeb63d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0e84574b-f7fe-45e9-bfe7-b405e3cba17e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-3bbeb63d"} 2026-06-11T20:34:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "openshift-service-ca.crt", "reconcileID": "7ad445af-21e3-45df-bda2-915190f04574", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:34:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-trusted-ca-bundle", "reconcileID": "d40908d2-10b4-428e-85c7-2f7864cfab37", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:34:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "openshift-service-ca.crt", "reconcileID": "9656232b-32f3-436b-806a-affcf3244e96", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-f214a9f2", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "openshift-service-ca.crt", "reconcileID": "9656232b-32f3-436b-806a-affcf3244e96", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:34:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "51ca09aa-793c-4d13-8ef7-cb9623568415", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:34:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "openshift-service-ca.crt", "reconcileID": "2588a678-2629-4510-9186-36f026b6f975", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:34:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-trusted-ca-bundle", "reconcileID": "139f510a-01f1-46ff-812e-c0b288f53f9c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-f214a9f2", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:34:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-trusted-ca-bundle", "reconcileID": "139f510a-01f1-46ff-812e-c0b288f53f9c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:34:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76a0b9bf-f4a4-4cc4-b13e-0d59804d3fb7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "57ebae9c-5101-434e-9eb9-cdf7790aa248", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "57ebae9c-5101-434e-9eb9-cdf7790aa248", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-f214a9f2", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T20:35:08Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "57ebae9c-5101-434e-9eb9-cdf7790aa248", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-f214a9f2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-trusted-ca-bundle", "reconcileID": "36fc35f7-ed25-4ab4-bd9d-1a8e810dff6a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-trusted-ca-bundle", "reconcileID": "36fc35f7-ed25-4ab4-bd9d-1a8e810dff6a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "openshift-service-ca.crt", "reconcileID": "ad432039-6988-417b-94ad-b9ced2e503f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "openshift-service-ca.crt", "reconcileID": "ad432039-6988-417b-94ad-b9ced2e503f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67700f95-e090-40ed-9c83-c45db4ef2097", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-f214a9f2"} 2026-06-11T20:35:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-f214a9f2"}, "namespace": "e2e-mt-f214a9f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67700f95-e090-40ed-9c83-c45db4ef2097", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-f214a9f2"}