2026-06-11T23:28:48Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "982a594a-99b3-457f-a473-6ac34bfbcca2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-c2b05a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:28:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "odh-trusted-ca-bundle", "reconcileID": "ee8af719-d156-4146-b17f-bf094c93f0f8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-c2b05a"} 2026-06-11T23:28:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "odh-trusted-ca-bundle", "reconcileID": "ee8af719-d156-4146-b17f-bf094c93f0f8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-c2b05a"} 2026-06-11T23:28:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "openshift-service-ca.crt", "reconcileID": "5e78fcf1-4d1f-4238-ae62-4397ee5a1a36", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-c2b05a"} 2026-06-11T23:28:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "openshift-service-ca.crt", "reconcileID": "5e78fcf1-4d1f-4238-ae62-4397ee5a1a36", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-c2b05a"} 2026-06-11T23:28:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b3cdef8-28b8-4d5e-b8f8-c997524468aa", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-c2b05a"} 2026-06-11T23:28:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-c2b05a"}, "namespace": "e2e-webhook-test-c2b05a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b3cdef8-28b8-4d5e-b8f8-c997524468aa", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-c2b05a"} 2026-06-11T23:28:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "openshift-service-ca.crt", "reconcileID": "de1a4f90-a57e-4c06-ad81-3c9b6acce31f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-trusted-ca-bundle", "reconcileID": "d8ecfd11-4a3a-4827-b301-4f6af4a69ed2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "openshift-service-ca.crt", "reconcileID": "7fb1f443-9fe9-4111-82a3-faf2206eb637", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-a9d97b", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:28:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "openshift-service-ca.crt", "reconcileID": "7fb1f443-9fe9-4111-82a3-faf2206eb637", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "openshift-service-ca.crt", "reconcileID": "c22fd7a1-bb64-4314-b8a1-8276eb6585df", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b3323f3d-c362-4755-8d6a-149ca06ea9a2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-trusted-ca-bundle", "reconcileID": "c2583900-8610-45b8-bc95-101f54f68ca7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-a9d97b", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:28:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-trusted-ca-bundle", "reconcileID": "c2583900-8610-45b8-bc95-101f54f68ca7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "df5c5618-55db-4fc7-bfc7-2a6cb0356a82", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc13375a-f28c-428c-9e5d-2ab8eb973a2f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc13375a-f28c-428c-9e5d-2ab8eb973a2f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-a9d97b", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:28:54Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc13375a-f28c-428c-9e5d-2ab8eb973a2f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-a9d97b because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:28:54Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-trusted-ca-bundle", "reconcileID": "b0b69c37-6328-4020-aaac-a26c206a82e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-trusted-ca-bundle", "reconcileID": "b0b69c37-6328-4020-aaac-a26c206a82e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "openshift-service-ca.crt", "reconcileID": "734fe5a6-83fe-4d22-8f49-4750684d2b97", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "openshift-service-ca.crt", "reconcileID": "734fe5a6-83fe-4d22-8f49-4750684d2b97", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b410d2f6-3b9c-46bc-bb3c-ee659871a42e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:28:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-a9d97b"}, "namespace": "e2e-webhook-test-a9d97b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b410d2f6-3b9c-46bc-bb3c-ee659871a42e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-a9d97b"} 2026-06-11T23:38:26Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:38:26Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:38:34Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:38:34Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:43:37Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:43:37Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:43:40Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:43:40Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:45:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c996a721-5046-4d4e-b7ac-2d5e4fa13179", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c996a721-5046-4d4e-b7ac-2d5e4fa13179", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T23:45:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c996a721-5046-4d4e-b7ac-2d5e4fa13179", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "28d19011-3ce9-42bd-9130-8d034759735b", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "28d19011-3ce9-42bd-9130-8d034759735b", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T23:45:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "28d19011-3ce9-42bd-9130-8d034759735b", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "bae9f2cf-2544-494c-98bf-123a2e390466", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "bae9f2cf-2544-494c-98bf-123a2e390466", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T23:45:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "bae9f2cf-2544-494c-98bf-123a2e390466", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:59Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "11ee997d-5753-452a-aafb-7131372bfb75", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:45:59Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "11ee997d-5753-452a-aafb-7131372bfb75", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-11T23:45:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "11ee997d-5753-452a-aafb-7131372bfb75", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-11T23:46:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-64487bd1","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-64487bd1", "reconcileID": "ab50f64a-229f-4262-b538-b8b3cc30d788", "Gateway": "e2e-ait-64487bd1", "namespace": "openshift-ingress"} 2026-06-11T23:46:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-64487bd1","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-64487bd1", "reconcileID": "8470722a-4f15-43da-8a32-c2ac646a58fe", "Gateway": "e2e-ait-64487bd1", "namespace": "openshift-ingress"} 2026-06-11T23:46:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-64487bd1","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-64487bd1", "reconcileID": "d43d2494-75db-400a-8e4c-1d0fdcb5ebf1", "Gateway": "e2e-ait-64487bd1", "namespace": "openshift-ingress"} 2026-06-11T23:46:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-64487bd1","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-64487bd1", "reconcileID": "9ab35fcb-8c0e-48d8-89f1-b97e61673b16", "Gateway": "e2e-ait-64487bd1", "namespace": "openshift-ingress"} 2026-06-11T23:46:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "openshift-service-ca.crt", "reconcileID": "b46d3716-9b42-4c10-a1c9-9a8d5494d688", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "openshift-service-ca.crt", "reconcileID": "0776e1af-887b-4bf1-843f-b49526a8edbd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-64487bd1", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:01Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "openshift-service-ca.crt", "reconcileID": "0776e1af-887b-4bf1-843f-b49526a8edbd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-trusted-ca-bundle", "reconcileID": "2e0ed5e0-8f94-4f22-b5f7-26082c8b5c92", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7e801ca3-6cc2-45a1-8bbc-98edabf43c2c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "openshift-service-ca.crt", "reconcileID": "33408950-dc82-4154-8325-0b104bfeea82", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-trusted-ca-bundle", "reconcileID": "cef65154-215d-42b6-86a5-44749934c39c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-64487bd1", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:01Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-trusted-ca-bundle", "reconcileID": "cef65154-215d-42b6-86a5-44749934c39c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7db9ac6d-0451-46a8-bd08-acf87177bde4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-64487bd1","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-64487bd1", "reconcileID": "38c0e65e-e4cb-4931-9736-61e1e0fd84f9", "Gateway": "e2e-ait-64487bd1", "namespace": "openshift-ingress"} 2026-06-11T23:46:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-64487bd1","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-64487bd1", "reconcileID": "54b4519b-6322-4ae6-9a98-b6b39a9cc8cb", "Gateway": "e2e-ait-64487bd1", "namespace": "openshift-ingress"} 2026-06-11T23:46:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bdc14692-1ea1-4d98-b921-00866bf91cea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bdc14692-1ea1-4d98-b921-00866bf91cea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-64487bd1", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bdc14692-1ea1-4d98-b921-00866bf91cea", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-64487bd1 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:46:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-trusted-ca-bundle", "reconcileID": "1970f2d3-694f-46b7-afd7-d44ddefced54", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-trusted-ca-bundle", "reconcileID": "1970f2d3-694f-46b7-afd7-d44ddefced54", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "openshift-service-ca.crt", "reconcileID": "90894386-dcb8-4c46-88c5-59111fcf4f9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "openshift-service-ca.crt", "reconcileID": "90894386-dcb8-4c46-88c5-59111fcf4f9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bd261f79-c052-4350-bf0c-532f6f0dfdc7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-64487bd1"}, "namespace": "e2e-aitenant-64487bd1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bd261f79-c052-4350-bf0c-532f6f0dfdc7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-64487bd1"} 2026-06-11T23:46:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "71ffa4ea-f3cb-41e8-852f-add434f0c148", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "52cfee39-585b-42ba-a1d8-1cefd4184ad3", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "e100f608-dc47-412b-9c48-54f3ba4f4ff1", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "aa7f213d-d715-4bf0-9726-c8d6406ee346", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "3c369672-9234-4efb-89ab-a3353a168078", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "openshift-service-ca.crt", "reconcileID": "3546b0ba-8107-4eb3-a24f-d58bec7520a3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-trusted-ca-bundle", "reconcileID": "fc69b455-9dbd-44bc-a652-72d569e571da", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "openshift-service-ca.crt", "reconcileID": "f60d0c8a-ceb3-4b64-b022-9b1ad5dd2433", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-9a9edf26", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "openshift-service-ca.crt", "reconcileID": "f60d0c8a-ceb3-4b64-b022-9b1ad5dd2433", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c52421ab-3429-4313-98b4-d3bc60bc57ff", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "openshift-service-ca.crt", "reconcileID": "9044d5fe-f862-4bf6-b914-9aa2716c44ca", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-trusted-ca-bundle", "reconcileID": "0fd7f811-6360-47e4-b999-e284f3d27274", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-9a9edf26", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-trusted-ca-bundle", "reconcileID": "0fd7f811-6360-47e4-b999-e284f3d27274", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "40ff8d2e-4801-467c-b528-856931f6e8fe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "cf38b4ae-54b3-4482-bc22-47ec4e24e5cc", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-9a9edf26","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-9a9edf26", "reconcileID": "eeb7ca9a-fc2e-495d-ae64-123524387704", "Gateway": "e2e-ait-9a9edf26", "namespace": "openshift-ingress"} 2026-06-11T23:46:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a7653ff1-be62-47f2-b4a1-be34554de106", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a7653ff1-be62-47f2-b4a1-be34554de106", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-9a9edf26", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:27Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a7653ff1-be62-47f2-b4a1-be34554de106", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-9a9edf26 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:46:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-trusted-ca-bundle", "reconcileID": "2008a75f-1a61-4a3b-bae2-1e4a0a56bce8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-trusted-ca-bundle", "reconcileID": "2008a75f-1a61-4a3b-bae2-1e4a0a56bce8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "openshift-service-ca.crt", "reconcileID": "b4780f4f-846c-4382-967e-61c9d609eb0e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "openshift-service-ca.crt", "reconcileID": "b4780f4f-846c-4382-967e-61c9d609eb0e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67b93d07-3b37-4246-9383-b69a8bb764a2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-9a9edf26"}, "namespace": "e2e-aitenant-9a9edf26", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67b93d07-3b37-4246-9383-b69a8bb764a2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-9a9edf26"} 2026-06-11T23:46:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-810bd9e5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-810bd9e5", "reconcileID": "af6842cf-1612-4634-80ad-d025d36da16c", "Gateway": "e2e-invalid-810bd9e5", "namespace": "openshift-ingress"} 2026-06-11T23:46:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-810bd9e5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-810bd9e5", "reconcileID": "8c91a4c6-d51c-4abf-adf5-20e07f7b464d", "Gateway": "e2e-invalid-810bd9e5", "namespace": "openshift-ingress"} 2026-06-11T23:46:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-810bd9e5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-810bd9e5", "reconcileID": "60ed24e8-fae4-473d-9a26-a86ae61b9b8b", "Gateway": "e2e-invalid-810bd9e5", "namespace": "openshift-ingress"} 2026-06-11T23:46:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-810bd9e5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-810bd9e5", "reconcileID": "43017aa6-a36a-4f41-ba86-370c1387028a", "Gateway": "e2e-invalid-810bd9e5", "namespace": "openshift-ingress"} 2026-06-11T23:46:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-invalid-810bd9e5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-invalid-810bd9e5", "reconcileID": "98f9ef1d-6220-4a4a-921e-0ce95427c643", "Gateway": "e2e-invalid-810bd9e5", "namespace": "openshift-ingress"} 2026-06-11T23:46:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "openshift-service-ca.crt", "reconcileID": "505ee252-d2c3-4dab-b9bb-f470b43e8e08", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-trusted-ca-bundle", "reconcileID": "f2c66530-6f33-47e0-9b67-5ef5a419ca92", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "openshift-service-ca.crt", "reconcileID": "a1dd6101-705d-4a44-bdf6-ac305194fee6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-a3743210", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "openshift-service-ca.crt", "reconcileID": "a1dd6101-705d-4a44-bdf6-ac305194fee6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "openshift-service-ca.crt", "reconcileID": "fe4fc3ad-2c0b-4972-921a-bc59c6584d1c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ffbe596d-d447-4211-bc4d-dd8b2a6eb6f1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-trusted-ca-bundle", "reconcileID": "6cf289ce-b67d-4a97-bfaa-87c50bb73f8b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-a3743210", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-trusted-ca-bundle", "reconcileID": "6cf289ce-b67d-4a97-bfaa-87c50bb73f8b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9cd62574-0888-4d06-9c0b-90e6c5682405", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c96fc7ed-a0cc-4eed-859f-2b8a09569923", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c96fc7ed-a0cc-4eed-859f-2b8a09569923", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-a3743210", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:35Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c96fc7ed-a0cc-4eed-859f-2b8a09569923", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-a3743210 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:46:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-trusted-ca-bundle", "reconcileID": "66c3dbb3-972c-4d5f-a4d0-0a79be13b840", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-trusted-ca-bundle", "reconcileID": "66c3dbb3-972c-4d5f-a4d0-0a79be13b840", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "openshift-service-ca.crt", "reconcileID": "a0284d18-73de-44a6-8017-9786d8b21061", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "openshift-service-ca.crt", "reconcileID": "a0284d18-73de-44a6-8017-9786d8b21061", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6bc0abae-3cd6-4cfd-910a-74049d0ccb41", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-a3743210"}, "namespace": "e2e-mt-a3743210", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6bc0abae-3cd6-4cfd-910a-74049d0ccb41", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-a3743210"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "openshift-service-ca.crt", "reconcileID": "a693a64c-b5df-4913-8b5a-e13c0cd7fe7d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-trusted-ca-bundle", "reconcileID": "744e4138-e59f-47cf-b296-593c78a3fb70", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:46:35Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "openshift-service-ca.crt", "reconcileID": "89639e59-e3ae-4c11-a939-4e83377ed356", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9fb13f2e", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:35Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "openshift-service-ca.crt", "reconcileID": "89639e59-e3ae-4c11-a939-4e83377ed356", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "openshift-service-ca.crt", "reconcileID": "61cf3ef9-2e8c-4c11-a2f6-f61cf4469436", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "54c36ace-7065-46a8-8306-cf186d63a6de", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:46:35Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-trusted-ca-bundle", "reconcileID": "89a24f79-78b7-49c6-8f90-dcef63eaf300", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9fb13f2e", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:46:35Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-trusted-ca-bundle", "reconcileID": "89a24f79-78b7-49c6-8f90-dcef63eaf300", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:46:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0a4b36b7-b0ab-48e7-9355-d00182c39cc3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c4f373bb-e434-4eea-8521-abf04d1f1430", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c4f373bb-e434-4eea-8521-abf04d1f1430", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9fb13f2e", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:08Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c4f373bb-e434-4eea-8521-abf04d1f1430", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-9fb13f2e because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:47:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-trusted-ca-bundle", "reconcileID": "c1c57e59-5ee3-45f3-b282-accf25441f44", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-trusted-ca-bundle", "reconcileID": "c1c57e59-5ee3-45f3-b282-accf25441f44", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "openshift-service-ca.crt", "reconcileID": "a8039d84-6008-4e3e-94da-ccf4253cd635", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "openshift-service-ca.crt", "reconcileID": "a8039d84-6008-4e3e-94da-ccf4253cd635", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3108aaaf-1929-43e5-8de3-b8d6548eb710", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9fb13f2e"}, "namespace": "e2e-mt-9fb13f2e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3108aaaf-1929-43e5-8de3-b8d6548eb710", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9fb13f2e"} 2026-06-11T23:47:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "openshift-service-ca.crt", "reconcileID": "f88ccd23-6f1b-40a7-a2ba-ecf46eeb77b9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-trusted-ca-bundle", "reconcileID": "c4d56403-5482-41ce-83a8-f31290b95fa7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "openshift-service-ca.crt", "reconcileID": "7710cc92-8639-4931-93be-c8a412d1a4b3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-aeae874d", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "openshift-service-ca.crt", "reconcileID": "7710cc92-8639-4931-93be-c8a412d1a4b3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f3ce3b2-b96e-46bf-8405-432a07296329", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "openshift-service-ca.crt", "reconcileID": "5a464cd0-8747-470a-90fd-123e09531afb", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-trusted-ca-bundle", "reconcileID": "c6996b7d-f1a8-4217-80b2-9201413bcc9c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-aeae874d", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-trusted-ca-bundle", "reconcileID": "c6996b7d-f1a8-4217-80b2-9201413bcc9c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "867ab752-7ef1-4708-b60a-35760402f8bc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "743720ba-4295-479d-a6e2-6e3711331567", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "743720ba-4295-479d-a6e2-6e3711331567", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-aeae874d", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "743720ba-4295-479d-a6e2-6e3711331567", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-aeae874d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:47:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-trusted-ca-bundle", "reconcileID": "4d9b313c-3a2e-429d-94e8-bd2cf1479215", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-trusted-ca-bundle", "reconcileID": "4d9b313c-3a2e-429d-94e8-bd2cf1479215", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "openshift-service-ca.crt", "reconcileID": "81cfb3e5-cf8a-4407-b43b-47ace2851d41", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "openshift-service-ca.crt", "reconcileID": "81cfb3e5-cf8a-4407-b43b-47ace2851d41", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "48f954ed-bfcd-442f-a224-7f3156aaad0e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-aeae874d"}, "namespace": "e2e-unlabeled-aeae874d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "48f954ed-bfcd-442f-a224-7f3156aaad0e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-aeae874d"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "openshift-service-ca.crt", "reconcileID": "d14dd733-87f9-47a1-8cc0-89d17c63337c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "openshift-service-ca.crt", "reconcileID": "b50d5d29-aa97-4cb3-9d37-5dda2311486e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ab4f2292", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:30Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "openshift-service-ca.crt", "reconcileID": "b50d5d29-aa97-4cb3-9d37-5dda2311486e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-trusted-ca-bundle", "reconcileID": "37d716d8-ab4f-4c35-b938-cd17de2fb474", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7b8cb30f-48a3-4d2e-90a0-2da6769d4006", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "openshift-service-ca.crt", "reconcileID": "4e0b8977-cd66-4c9c-bcc0-22f8f8208988", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-trusted-ca-bundle", "reconcileID": "e7b2472f-b6b7-48f8-b141-d6a1c6b2bf87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ab4f2292", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-trusted-ca-bundle", "reconcileID": "e7b2472f-b6b7-48f8-b141-d6a1c6b2bf87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "52e20315-9a0d-49be-8e8f-2eee669c9de5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f1c623d9-8721-4e2b-809d-44d688a375c9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f1c623d9-8721-4e2b-809d-44d688a375c9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ab4f2292", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:47Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f1c623d9-8721-4e2b-809d-44d688a375c9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-ab4f2292 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:47:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-trusted-ca-bundle", "reconcileID": "ee61b527-9c47-434f-987e-ec4ad7bfaf12", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-trusted-ca-bundle", "reconcileID": "ee61b527-9c47-434f-987e-ec4ad7bfaf12", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "openshift-service-ca.crt", "reconcileID": "5fa36738-d73c-4ac0-879a-cb1bbb280dc5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "openshift-service-ca.crt", "reconcileID": "5fa36738-d73c-4ac0-879a-cb1bbb280dc5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "152be115-128d-4bb0-aefd-dfb04e9eaa50", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ab4f2292"}, "namespace": "e2e-mt-ab4f2292", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "152be115-128d-4bb0-aefd-dfb04e9eaa50", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ab4f2292"} 2026-06-11T23:47:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "openshift-service-ca.crt", "reconcileID": "28da0c4b-4a8d-4b9f-acdc-1fb7c5104661", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-trusted-ca-bundle", "reconcileID": "8eba8438-a682-4f73-a2ed-38d6ba053ddb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:48Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "openshift-service-ca.crt", "reconcileID": "c64879d3-a462-4386-84d3-fb39061a17a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-bd4d6343", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:48Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "openshift-service-ca.crt", "reconcileID": "c64879d3-a462-4386-84d3-fb39061a17a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "748fbef0-e778-449a-9876-d5a6967322cc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "openshift-service-ca.crt", "reconcileID": "3f399d87-43b5-4726-8638-d094d9555c88", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:48Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-trusted-ca-bundle", "reconcileID": "7e3641e1-bbff-4e60-a487-494649e53222", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-bd4d6343", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:48Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-trusted-ca-bundle", "reconcileID": "7e3641e1-bbff-4e60-a487-494649e53222", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fa547b6f-ade6-4cb8-9f78-e7becec9b827", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "30dda8c5-c9d7-4bb3-8f0e-8c31acfc7200", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "30dda8c5-c9d7-4bb3-8f0e-8c31acfc7200", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-bd4d6343", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:55Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "30dda8c5-c9d7-4bb3-8f0e-8c31acfc7200", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-bd4d6343 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:47:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-trusted-ca-bundle", "reconcileID": "e78b5b46-d3b5-4268-852d-f1591588e91e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-trusted-ca-bundle", "reconcileID": "e78b5b46-d3b5-4268-852d-f1591588e91e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "openshift-service-ca.crt", "reconcileID": "6b97d384-82ea-4aaa-9461-d6006559d163", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "openshift-service-ca.crt", "reconcileID": "6b97d384-82ea-4aaa-9461-d6006559d163", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5c3656a4-320a-4915-8651-467cb76d5cec", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-bd4d6343"}, "namespace": "e2e-mt-bd4d6343", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5c3656a4-320a-4915-8651-467cb76d5cec", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-bd4d6343"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "openshift-service-ca.crt", "reconcileID": "8cf94c86-58a1-4c61-9c02-51c48064b35d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-trusted-ca-bundle", "reconcileID": "aec829c7-4434-43f1-9a4e-ef75cbee2409", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "openshift-service-ca.crt", "reconcileID": "bf99bd30-f138-4dde-8a93-3e994c6c6b36", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-90c38478", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:55Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "openshift-service-ca.crt", "reconcileID": "bf99bd30-f138-4dde-8a93-3e994c6c6b36", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "openshift-service-ca.crt", "reconcileID": "bcb50e67-9569-4ffe-ba6b-a15930f5adb6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81700a66-253d-497e-85f1-8cc5c21565ee", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-trusted-ca-bundle", "reconcileID": "7be2d0a3-dc06-4ce5-b733-239ec838028b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-90c38478", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:55Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-trusted-ca-bundle", "reconcileID": "7be2d0a3-dc06-4ce5-b733-239ec838028b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a83acc6b-43d3-475d-836c-c21d87d95cf6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:47:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "openshift-service-ca.crt", "reconcileID": "95be157f-dbb0-4e68-a688-7378d301d382", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:47:56Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "openshift-service-ca.crt", "reconcileID": "9a8ee4f8-714a-489e-be35-f6c1fc009a19", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-89698c20", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:56Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "openshift-service-ca.crt", "reconcileID": "9a8ee4f8-714a-489e-be35-f6c1fc009a19", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:47:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-trusted-ca-bundle", "reconcileID": "c6723c74-3267-4d68-a122-56bd9b613bd4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:47:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b7b5fd45-a3b0-4b87-b374-41c9d02792dc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:47:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "openshift-service-ca.crt", "reconcileID": "186bb3f2-9023-4807-900f-8c6257975103", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:47:56Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-trusted-ca-bundle", "reconcileID": "bdf708e8-633c-4c5e-bad9-0772afd4bcb0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-89698c20", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:47:56Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-trusted-ca-bundle", "reconcileID": "bdf708e8-633c-4c5e-bad9-0772afd4bcb0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:47:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e046571-322a-498f-bc01-e4cd1e4db309", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a07dea14-5625-4ce6-a60c-de14e57f1058", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a07dea14-5625-4ce6-a60c-de14e57f1058", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-90c38478", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:03Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a07dea14-5625-4ce6-a60c-de14e57f1058", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-90c38478 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:48:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-trusted-ca-bundle", "reconcileID": "df93ca20-710e-40c9-a2ac-598e73e46b59", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-trusted-ca-bundle", "reconcileID": "df93ca20-710e-40c9-a2ac-598e73e46b59", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "openshift-service-ca.crt", "reconcileID": "e967dfd2-ed51-4098-8b83-82765164375a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "openshift-service-ca.crt", "reconcileID": "e967dfd2-ed51-4098-8b83-82765164375a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "73e279cb-48ff-41e7-8564-6c4869dd244b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-90c38478"}, "namespace": "e2e-mt-90c38478", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "73e279cb-48ff-41e7-8564-6c4869dd244b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-90c38478"} 2026-06-11T23:48:10Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "447b11d7-bd93-4009-8495-03a934a15e5d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "447b11d7-bd93-4009-8495-03a934a15e5d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-89698c20", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:10Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "447b11d7-bd93-4009-8495-03a934a15e5d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-89698c20 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:48:10Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-trusted-ca-bundle", "reconcileID": "5cdf1ee2-d646-4af6-a3e4-75cdb51993f9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-trusted-ca-bundle", "reconcileID": "5cdf1ee2-d646-4af6-a3e4-75cdb51993f9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:10Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "openshift-service-ca.crt", "reconcileID": "74a26612-d834-4011-99a4-34474d22a1dc", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "openshift-service-ca.crt", "reconcileID": "74a26612-d834-4011-99a4-34474d22a1dc", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:10Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1a24fa78-d7b3-4c2d-931c-7ec70fd029dc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-89698c20"}, "namespace": "e2e-mt-89698c20", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1a24fa78-d7b3-4c2d-931c-7ec70fd029dc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-89698c20"} 2026-06-11T23:48:11Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "947fe735-30e5-42fb-a28a-cbe0a0ba75c2", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:11Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "1531fe3c-ecfe-4667-93a9-e53ceab96ecb", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:11Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "6b7d112f-83f7-4414-bdf5-319f6273fa47", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:11Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "165a5c17-9b5a-4052-b3fa-65340d201b1e", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:11Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "0fc03809-2797-46a1-ab62-747a459e3453", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "37d2f3ec-0ce6-434a-9d87-96da850f6aba", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "a0f3ab76-ce88-4438-898e-5d37e517e68d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "f9c47a91-773a-4665-9d08-f67391962a2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-7c22f708", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "f9c47a91-773a-4665-9d08-f67391962a2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "46dc171e-6d28-4cf1-ae24-539755971c8e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "e163d837-d7fb-4b16-b094-4311b2b36c79", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "27a5e498-88fb-4954-8728-73efd2a683ac", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-7c22f708", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "27a5e498-88fb-4954-8728-73efd2a683ac", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "efe69962-0b5d-4041-bad8-f3cc8dc6826f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:13Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "a3a651f4-f790-4bc1-bc21-8f62e464f9ed", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7c22f708","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7c22f708", "reconcileID": "c7764cc5-0d2a-409f-9470-c3c8cdb6993e", "Gateway": "e2e-mt-7c22f708", "namespace": "openshift-ingress"} 2026-06-11T23:48:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "6abdae46-65fa-411e-90d8-c0dd4d310f52", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "fc1d59a8-9dbc-4ea9-b7e2-292b0a247f57", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "b729e18c-1cb2-4e08-9b8d-987aa73b2cc9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-7c22f708", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "b729e18c-1cb2-4e08-9b8d-987aa73b2cc9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "040505c9-d868-4a70-bb2a-4028235840ae", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "22d2c228-b744-45a3-9886-cbff9a533e53", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "6c258536-3cea-4894-af9c-4f565deac60a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-7c22f708", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "6c258536-3cea-4894-af9c-4f565deac60a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d3597f37-9ddc-4f9b-a6c4-626dff72bbfb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "362c7471-0569-4e11-9eb9-ef3dcc6c4ed5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "362c7471-0569-4e11-9eb9-ef3dcc6c4ed5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-7c22f708", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:24Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "362c7471-0569-4e11-9eb9-ef3dcc6c4ed5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-other-7c22f708 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:48:24Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "97079d91-d9c0-4004-8578-c4b66b77a8dd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "97079d91-d9c0-4004-8578-c4b66b77a8dd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "ecf476ea-f857-4c2e-9f70-33975cdff3c1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "ecf476ea-f857-4c2e-9f70-33975cdff3c1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f490b22-3649-44d4-8a76-ea64ba6cd73d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-other-7c22f708"}, "namespace": "e2e-mt-other-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f490b22-3649-44d4-8a76-ea64ba6cd73d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-other-7c22f708"} 2026-06-11T23:48:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af29a439-09b7-45df-a0f0-cd4d59d35649", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af29a439-09b7-45df-a0f0-cd4d59d35649", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-7c22f708", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af29a439-09b7-45df-a0f0-cd4d59d35649", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-7c22f708 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:48:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "707352d2-0c67-4b02-9307-20e477793a2f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-trusted-ca-bundle", "reconcileID": "707352d2-0c67-4b02-9307-20e477793a2f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "81bc8a48-82c0-498d-acc1-7eb3b2d1aa68", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "openshift-service-ca.crt", "reconcileID": "81bc8a48-82c0-498d-acc1-7eb3b2d1aa68", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c4c46c3-d736-497d-ba3f-11c4f2448968", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-7c22f708"}, "namespace": "e2e-mt-7c22f708", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c4c46c3-d736-497d-ba3f-11c4f2448968", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-7c22f708"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "openshift-service-ca.crt", "reconcileID": "190bd282-ca75-4417-b83a-bb718de151c0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-trusted-ca-bundle", "reconcileID": "4f65adbb-99c9-4d22-a7b6-d942f8239ab3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "openshift-service-ca.crt", "reconcileID": "a372fa3f-e2ce-4728-9a45-c8134e44685e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8b9de8c8", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:30Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "openshift-service-ca.crt", "reconcileID": "a372fa3f-e2ce-4728-9a45-c8134e44685e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "openshift-service-ca.crt", "reconcileID": "b90ecb11-b692-41f0-b215-ca9200fc0306", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "edd8e73d-ad68-4ee5-ba77-f3c9494fea70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-trusted-ca-bundle", "reconcileID": "40a7b406-8dd4-40d3-93b8-a0c375cc4eb4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-trusted-ca-bundle", "reconcileID": "40a7b406-8dd4-40d3-93b8-a0c375cc4eb4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af3347b1-8878-4ccc-917d-db8929eff225", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dea99f2a-cc05-40e2-8c0c-cf89e8b46ff0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dea99f2a-cc05-40e2-8c0c-cf89e8b46ff0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:36Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dea99f2a-cc05-40e2-8c0c-cf89e8b46ff0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-8b9de8c8 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:48:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-trusted-ca-bundle", "reconcileID": "537198ba-cbfd-4cd8-9533-6b968f89702d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-trusted-ca-bundle", "reconcileID": "537198ba-cbfd-4cd8-9533-6b968f89702d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "openshift-service-ca.crt", "reconcileID": "89e18d1a-c6df-45a0-8e6c-daa13eec6632", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "openshift-service-ca.crt", "reconcileID": "89e18d1a-c6df-45a0-8e6c-daa13eec6632", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ddb79cc7-05f0-4f48-9c0a-01c83f97c1a3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-8b9de8c8"}, "namespace": "e2e-webhook-sub-8b9de8c8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ddb79cc7-05f0-4f48-9c0a-01c83f97c1a3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-8b9de8c8"} 2026-06-11T23:48:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "openshift-service-ca.crt", "reconcileID": "4d6ebd9f-bf0c-4810-bc1a-af6859d41535", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-trusted-ca-bundle", "reconcileID": "8decd508-13d2-42f6-9154-72b731e5fc7b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "openshift-service-ca.crt", "reconcileID": "48a0c977-9f9d-4b55-98be-a393531a6561", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1a88c686", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:37Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "openshift-service-ca.crt", "reconcileID": "48a0c977-9f9d-4b55-98be-a393531a6561", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "495bc442-284a-4f29-957e-19b5bb9630cd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "openshift-service-ca.crt", "reconcileID": "05814d61-d5ec-4ec2-937a-5be817525e44", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-trusted-ca-bundle", "reconcileID": "f3783ec1-3389-4ce8-9997-99eea6969173", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:37Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-trusted-ca-bundle", "reconcileID": "f3783ec1-3389-4ce8-9997-99eea6969173", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aecb9c01-c5a0-457e-ad44-5439e48405d3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5f6916e6-450a-434b-a1ba-7c42cee4ee02", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5f6916e6-450a-434b-a1ba-7c42cee4ee02", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:43Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5f6916e6-450a-434b-a1ba-7c42cee4ee02", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-1a88c686 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:48:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-trusted-ca-bundle", "reconcileID": "67aa4b0f-a9c0-498c-a711-fd448ce2365c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-trusted-ca-bundle", "reconcileID": "67aa4b0f-a9c0-498c-a711-fd448ce2365c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "openshift-service-ca.crt", "reconcileID": "a58074b5-bfc8-4e68-b900-d1ac4507f813", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "openshift-service-ca.crt", "reconcileID": "a58074b5-bfc8-4e68-b900-d1ac4507f813", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5988437e-5b12-4c71-9cb7-afed24d7d9b3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1a88c686"}, "namespace": "e2e-webhook-auth-1a88c686", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5988437e-5b12-4c71-9cb7-afed24d7d9b3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1a88c686"} 2026-06-11T23:48:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "ce711ffc-c976-4c03-9bc2-29be7755233c", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:48:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "22a91ebd-8615-4adc-8880-c8638499046a", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:48:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "03dfbcd3-aacf-4540-afb4-d680ca728474", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:48:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "22d07813-f1ec-43aa-b074-f9fd7b6ae6a4", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:48:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "02c21d48-c209-4686-92e3-d3631481c873", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:48:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "openshift-service-ca.crt", "reconcileID": "826a9c96-e472-467d-9b9e-161d7a5d2235", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-trusted-ca-bundle", "reconcileID": "e0c468b3-afbf-4583-a90a-7cb674a36c5e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:54Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "openshift-service-ca.crt", "reconcileID": "1c1e5422-c845-4134-8a46-d41c3e62c996", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-aecf84b0", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:54Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "openshift-service-ca.crt", "reconcileID": "1c1e5422-c845-4134-8a46-d41c3e62c996", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "80a3ce85-7aed-4c42-bb3f-44b7b31bf27d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "openshift-service-ca.crt", "reconcileID": "7d821d60-2dae-4098-a438-a9fcecbdeb32", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:54Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-trusted-ca-bundle", "reconcileID": "7b747751-3552-4ced-aaf2-c8e2d7b02a50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-aecf84b0", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:48:54Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-trusted-ca-bundle", "reconcileID": "7b747751-3552-4ced-aaf2-c8e2d7b02a50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b4ee0f91-1ec0-4020-b320-6a59a48854fb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:48:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "a4f2e86d-69c6-486c-9226-b83e7eff1796", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:48:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-aecf84b0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-aecf84b0", "reconcileID": "b228d13a-66be-4daa-9988-4fb98386668d", "Gateway": "e2e-mt-aecf84b0", "namespace": "openshift-ingress"} 2026-06-11T23:49:49Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:49:49Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:49:50Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:49:50Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-11T23:51:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d8c1d08e-34e8-4585-a2b5-ff8efe309000", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d8c1d08e-34e8-4585-a2b5-ff8efe309000", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-aecf84b0", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:09Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d8c1d08e-34e8-4585-a2b5-ff8efe309000", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-aecf84b0 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:51:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-trusted-ca-bundle", "reconcileID": "83b62985-a36b-4ad0-905d-b741923fb1b6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-trusted-ca-bundle", "reconcileID": "83b62985-a36b-4ad0-905d-b741923fb1b6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "openshift-service-ca.crt", "reconcileID": "59cfde42-38c9-4e2a-a35d-1fdd78184df9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "openshift-service-ca.crt", "reconcileID": "59cfde42-38c9-4e2a-a35d-1fdd78184df9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9f3855d7-2221-44b9-bb6c-80840eb65d70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-aecf84b0"}, "namespace": "e2e-mt-aecf84b0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9f3855d7-2221-44b9-bb6c-80840eb65d70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-aecf84b0"} 2026-06-11T23:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "openshift-service-ca.crt", "reconcileID": "450e7b72-1ed1-44c8-b274-c153ca0fe95b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-trusted-ca-bundle", "reconcileID": "71db856d-f20f-468d-9619-1e7f977695fb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9550c8b4", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-trusted-ca-bundle", "reconcileID": "71db856d-f20f-468d-9619-1e7f977695fb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "openshift-service-ca.crt", "reconcileID": "2dcf5982-062e-47ad-bc9a-b53771150f98", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74535beb-369c-46c3-b9c4-7f40deb9529c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-trusted-ca-bundle", "reconcileID": "ec976efa-1ddb-4ea3-90be-c9bd3e063904", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9550c8b4", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-trusted-ca-bundle", "reconcileID": "ec976efa-1ddb-4ea3-90be-c9bd3e063904", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "01c09beb-b426-4784-8094-b34d9e74861f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "openshift-service-ca.crt", "reconcileID": "02d1cf5b-2b28-4f2c-a0d4-ece9e9bf9254", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-trusted-ca-bundle", "reconcileID": "db32f785-5493-4f19-9e50-92f456695e06", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "openshift-service-ca.crt", "reconcileID": "e20f0966-bd4f-40cf-896e-a938aa47be65", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-15415ef0", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "openshift-service-ca.crt", "reconcileID": "e20f0966-bd4f-40cf-896e-a938aa47be65", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bde1200e-1f1f-4e31-a6d3-bbc465824203", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "openshift-service-ca.crt", "reconcileID": "b96cf6b9-5e0c-4626-9eda-9cec75a390f2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-trusted-ca-bundle", "reconcileID": "6ef963d1-c93a-461c-8770-f437ad870013", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-15415ef0", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-trusted-ca-bundle", "reconcileID": "6ef963d1-c93a-461c-8770-f437ad870013", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "50e6a4b4-6fe7-47e3-a5f7-f1377a9eae88", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fe8e24fa-c4ec-4f41-abf8-0c47a72d1f5b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fe8e24fa-c4ec-4f41-abf8-0c47a72d1f5b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9550c8b4", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fe8e24fa-c4ec-4f41-abf8-0c47a72d1f5b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-9550c8b4 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:51:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-trusted-ca-bundle", "reconcileID": "98b5baf4-6a1d-4dc4-af5c-29e47f1d25c4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-trusted-ca-bundle", "reconcileID": "98b5baf4-6a1d-4dc4-af5c-29e47f1d25c4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "openshift-service-ca.crt", "reconcileID": "147796ec-a221-4e28-8c17-ed4b59ee5ad2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "openshift-service-ca.crt", "reconcileID": "147796ec-a221-4e28-8c17-ed4b59ee5ad2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b7b8a5a-d173-4096-8b28-9297829f0f67", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-9550c8b4"}, "namespace": "e2e-mt-9550c8b4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b7b8a5a-d173-4096-8b28-9297829f0f67", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-9550c8b4"} 2026-06-11T23:51:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6ec46ab0-8d76-4791-99e6-194f8faec51a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6ec46ab0-8d76-4791-99e6-194f8faec51a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-15415ef0", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:25Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6ec46ab0-8d76-4791-99e6-194f8faec51a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-15415ef0 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:51:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-trusted-ca-bundle", "reconcileID": "567d7976-f753-47e7-93a4-aa88c061f7fa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-trusted-ca-bundle", "reconcileID": "567d7976-f753-47e7-93a4-aa88c061f7fa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "openshift-service-ca.crt", "reconcileID": "341bbf90-2d1e-4265-b0bb-9032802efb87", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "openshift-service-ca.crt", "reconcileID": "341bbf90-2d1e-4265-b0bb-9032802efb87", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c04a52cd-5df3-4024-9556-1e0f92483dff", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-15415ef0"}, "namespace": "e2e-mt-15415ef0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c04a52cd-5df3-4024-9556-1e0f92483dff", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-15415ef0"} 2026-06-11T23:51:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "openshift-service-ca.crt", "reconcileID": "9a9acf15-c691-4c07-9ddf-8464e2babe15", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:51:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-trusted-ca-bundle", "reconcileID": "d75735d3-1999-40bf-890b-073a257e60fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:51:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "openshift-service-ca.crt", "reconcileID": "e533b3d7-c093-47ee-9bde-9d41a7c017bb", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ffd73503", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:26Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "openshift-service-ca.crt", "reconcileID": "e533b3d7-c093-47ee-9bde-9d41a7c017bb", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:51:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "69df1bc4-1052-4988-9dbf-06729e63d3fc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:51:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "openshift-service-ca.crt", "reconcileID": "5bee7471-7eb2-4377-8d9e-226c92651c44", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:51:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-trusted-ca-bundle", "reconcileID": "87d44154-ee0e-4127-ba5f-a62dc16359b6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ffd73503", "update": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:51:26Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-trusted-ca-bundle", "reconcileID": "87d44154-ee0e-4127-ba5f-a62dc16359b6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:51:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "734fbe8c-ec70-4c97-9de7-acbc849e7e66", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dac69030-7e86-4e58-b3d4-6edc57fafd82", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dac69030-7e86-4e58-b3d4-6edc57fafd82", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ffd73503", "create": "odh-kserve-custom-ca-bundle"} 2026-06-11T23:52:03Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dac69030-7e86-4e58-b3d4-6edc57fafd82", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-mt-ffd73503 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-11T23:52:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-trusted-ca-bundle", "reconcileID": "7cd052ab-e627-46ce-9451-661a7c1536d1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-trusted-ca-bundle", "reconcileID": "7cd052ab-e627-46ce-9451-661a7c1536d1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "openshift-service-ca.crt", "reconcileID": "e2014667-37e4-492a-bafa-222db14aa13d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "openshift-service-ca.crt", "reconcileID": "e2014667-37e4-492a-bafa-222db14aa13d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aacfb427-fa01-4e2a-842e-7ec06445ed8d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ffd73503"} 2026-06-11T23:52:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-mt-ffd73503"}, "namespace": "e2e-mt-ffd73503", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aacfb427-fa01-4e2a-842e-7ec06445ed8d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-mt-ffd73503"}