sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:16:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-0dbbab33"}, "namespace": "ai-tenant-e2e-ait-0dbbab33", "name": "odh-trusted-ca-bundle", "reconcileID": "64ee16b9-4015-4e08-bb50-d3db52cf4a7e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-0dbbab33"}
2026-07-10T15:16:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-0dbbab33"}, "namespace": "ai-tenant-e2e-ait-0dbbab33", "name": "odh-trusted-ca-bundle", "reconcileID": "64ee16b9-4015-4e08-bb50-d3db52cf4a7e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-0dbbab33"}
2026-07-10T15:16:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-0dbbab33"}, "namespace": "ai-tenant-e2e-ait-0dbbab33", "name": "openshift-service-ca.crt", "reconcileID": "20bb7f1d-9252-482e-af6d-a0859b30c6a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-0dbbab33"}
2026-07-10T15:16:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-0dbbab33"}, "namespace": "ai-tenant-e2e-ait-0dbbab33", "name": "openshift-service-ca.crt", "reconcileID": "20bb7f1d-9252-482e-af6d-a0859b30c6a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-0dbbab33"}
2026-07-10T15:16:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-0dbbab33"}, "namespace": "ai-tenant-e2e-ait-0dbbab33", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a47d222-69db-428a-bf08-8427285222be", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-0dbbab33"}
2026-07-10T15:16:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-0dbbab33"}, "namespace": "ai-tenant-e2e-ait-0dbbab33", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a47d222-69db-428a-bf08-8427285222be", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-0dbbab33"}
2026-07-10T15:16:14Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "bdbe8401-379a-4203-b7b2-57ce34d8ff98", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:14Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "5f54d1a8-1189-4d28-9f93-07c1c5cfd63c", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:14Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "c11b7269-68ce-4255-b963-dcdc48fb3827", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:14Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "2856a72a-04cc-4558-8e69-362cb3731b47", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:14Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "edbc2f69-c117-49aa-88b5-4d6932965b4e", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "a189abc6-f158-4ddc-b221-8c081a78964e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-trusted-ca-bundle", "reconcileID": "4aa3a3c0-0ba6-4e91-9758-b40fd2390fa3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "67c2a812-93a2-4ecd-a344-383c2ded6c13", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "67c2a812-93a2-4ecd-a344-383c2ded6c13", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "dfa3edcc-4135-4ccf-b274-c58d982f0d0f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:15Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "dfa3edcc-4135-4ccf-b274-c58d982f0d0f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:16:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1f0bda6a-4536-4cc6-9ecb-6404f8a774de", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "c9e5e47f-495a-4e38-881b-e1b22463879b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-trusted-ca-bundle", "reconcileID": "b261e60c-6c9c-4de3-8883-a3fca05a410f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-trusted-ca-bundle", "reconcileID": "b261e60c-6c9c-4de3-8883-a3fca05a410f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a0c8c884-3e63-4a36-9baa-9aa3b62fa905", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:16Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "7358a1e1-e6d4-4051-a9d7-80facc03057a", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "dd12708c-eae6-435f-b4fa-c22959834716", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-10T15:16:18Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "7bed59d0-8fe6-43a6-b3ef-4a54a4f21257", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4ef8e887","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4ef8e887", "reconcileID": "dc0e4b34-6d2c-46a2-8f66-6cf432b29327", "Gateway": "e2e-ait-4ef8e887", "namespace": "openshift-ingress"}
2026-07-10T15:16:31Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "24acbfc7-87c2-486f-a723-21068e0628b8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:31Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "24acbfc7-87c2-486f-a723-21068e0628b8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:31Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "24acbfc7-87c2-486f-a723-21068e0628b8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ait-4ef8e887 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:16:31Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-trusted-ca-bundle", "reconcileID": "5d133170-ba69-4d61-be98-b0c6a6318576", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-trusted-ca-bundle", "reconcileID": "5d133170-ba69-4d61-be98-b0c6a6318576", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:31Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "889b0a8c-5956-4d88-be07-cea0ef2106d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "openshift-service-ca.crt", "reconcileID": "889b0a8c-5956-4d88-be07-cea0ef2106d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:31Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6dbfbb37-bd5e-442c-849c-dfa30872ce95", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-4ef8e887"}, "namespace": "ai-tenant-e2e-ait-4ef8e887", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6dbfbb37-bd5e-442c-849c-dfa30872ce95", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-4ef8e887"}
2026-07-10T15:16:33Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "47db2ee2-0728-42a2-84a1-f637d2f7912d", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:33Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "d51d8ce2-d24a-43a5-a429-f0efe8fd4a63", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:33Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "38e52ded-00b2-4951-aaa3-a18b7a493bd4", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:33Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "cb5af943-faad-4b60-94c6-a971a46c719e", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:33Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "d91e44a5-63cc-4c42-85dc-a1e73334051c", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:34Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "4d13879f-7234-402f-84a7-8edc0097766d", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:34Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-trusted-ca-bundle", "reconcileID": "e124a9d0-380c-4b16-bacd-488cf6107308", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:34Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "openshift-service-ca.crt", "reconcileID": "e3dc39b9-04a2-4f46-a6e2-465cff965e32", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:34Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "openshift-service-ca.crt", "reconcileID": "4712010f-20b4-4eb6-8645-c4316d898c25", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-65757421", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:34Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "openshift-service-ca.crt", "reconcileID": "4712010f-20b4-4eb6-8645-c4316d898c25", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:34Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8ad3134a-94b3-40cb-9bcf-a02fdce9e5e0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:34Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "openshift-service-ca.crt", "reconcileID": "4d7c2b67-6da5-4beb-8a94-0a40d95a2ac1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:34Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-trusted-ca-bundle", "reconcileID": "ee826db9-330d-40ef-a9e3-6628c52c42df", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:34Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-trusted-ca-bundle", "reconcileID": "ee826db9-330d-40ef-a9e3-6628c52c42df", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:34Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d6b8632a-e510-472e-8a8e-969a30d41a02", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "2cd16142-04db-472e-9d75-b34367acf92d", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-10T15:16:36Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-65757421","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-65757421", "reconcileID": "9ed728dd-b2a1-4c3b-9169-a2ddc6f2dab7", "Gateway": "e2e-derive-65757421", "namespace": "openshift-ingress"}
2026-07-10T15:16:44Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f53e75fc-2145-4752-a207-d0dafbd3a985", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:44Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f53e75fc-2145-4752-a207-d0dafbd3a985", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:44Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f53e75fc-2145-4752-a207-d0dafbd3a985", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-derive-65757421 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:16:44Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-trusted-ca-bundle", "reconcileID": "87dbbae7-50b2-4522-abbc-bb7f6e6ec550", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-trusted-ca-bundle", "reconcileID": "87dbbae7-50b2-4522-abbc-bb7f6e6ec550", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:44Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "openshift-service-ca.crt", "reconcileID": "42a67537-8ff1-4d58-a57c-22609e89e4b7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "openshift-service-ca.crt", "reconcileID": "42a67537-8ff1-4d58-a57c-22609e89e4b7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:44Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd878369-c6e3-4e08-9f10-c978e7c6603a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-65757421"}, "namespace": "ai-tenant-e2e-derive-65757421", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd878369-c6e3-4e08-9f10-c978e7c6603a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-65757421"}
2026-07-10T15:16:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "openshift-service-ca.crt", "reconcileID": "80ecdda0-e4aa-422a-99ed-2accbf75d469", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-trusted-ca-bundle", "reconcileID": "e06f9161-7ad4-4e6c-9ec0-af84b0875784", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "openshift-service-ca.crt", "reconcileID": "2bb7ea10-d9f4-4c6d-92f8-16f164509ded", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-87dd6ff9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:45Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "openshift-service-ca.crt", "reconcileID": "2bb7ea10-d9f4-4c6d-92f8-16f164509ded", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6e84de76-e6a4-45f7-bdbb-55a71ea12249", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "openshift-service-ca.crt", "reconcileID": "400b631f-61f4-427a-b989-024e74292e41", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-trusted-ca-bundle", "reconcileID": "a030aeb3-aaaa-4f56-9baf-5d899959c6c6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:46Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-trusted-ca-bundle", "reconcileID": "a030aeb3-aaaa-4f56-9baf-5d899959c6c6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81863191-1b17-4b43-ba6e-5a6e6128cfd4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "13750227-5649-40ca-9694-1b4c4ada5641", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "13750227-5649-40ca-9694-1b4c4ada5641", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:54Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "13750227-5649-40ca-9694-1b4c4ada5641", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-87dd6ff9 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:16:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-trusted-ca-bundle", "reconcileID": "6ffa2768-006f-4733-bbff-09580cedf62d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-trusted-ca-bundle", "reconcileID": "6ffa2768-006f-4733-bbff-09580cedf62d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "openshift-service-ca.crt", "reconcileID": "5a7c2348-e8a6-48af-a24f-fbb500c50f03", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "openshift-service-ca.crt", "reconcileID": "5a7c2348-e8a6-48af-a24f-fbb500c50f03", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ad91d5e4-3e5f-4b30-9c6f-d0bf50df17be", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-87dd6ff9"}, "namespace": "ai-tenant-e2e-mt-87dd6ff9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ad91d5e4-3e5f-4b30-9c6f-d0bf50df17be", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-87dd6ff9"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "openshift-service-ca.crt", "reconcileID": "8356f7f6-cb4c-4b41-9550-31b1d12a0f0c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-trusted-ca-bundle", "reconcileID": "359753b0-38ca-4c82-816b-d5d3b3a06503", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:16:54Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "openshift-service-ca.crt", "reconcileID": "44662c84-51d7-48d7-8823-a830646bdff1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-27191095", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:54Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "openshift-service-ca.crt", "reconcileID": "44662c84-51d7-48d7-8823-a830646bdff1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f8353ef7-09ec-4e61-a31b-39ef468de62e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:16:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "openshift-service-ca.crt", "reconcileID": "bcd68a7a-a8e1-4c24-bb10-b6c754ed8746", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:16:55Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-trusted-ca-bundle", "reconcileID": "29fbfa3a-20c7-4168-b52a-d5d33b47c89a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:16:55Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-trusted-ca-bundle", "reconcileID": "29fbfa3a-20c7-4168-b52a-d5d33b47c89a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:16:55Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "75e3b255-70b2-48d8-97d4-86f85968acd7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf6c5a53-49f0-4124-8154-4cd1b2b197bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf6c5a53-49f0-4124-8154-4cd1b2b197bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:26Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf6c5a53-49f0-4124-8154-4cd1b2b197bb", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-27191095 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:17:26Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-trusted-ca-bundle", "reconcileID": "f7b5420c-3670-417a-a384-8d5fdd669a56", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-trusted-ca-bundle", "reconcileID": "f7b5420c-3670-417a-a384-8d5fdd669a56", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "openshift-service-ca.crt", "reconcileID": "b864f948-3844-44a0-9da6-08926d7781c5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "openshift-service-ca.crt", "reconcileID": "b864f948-3844-44a0-9da6-08926d7781c5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3ddedcd0-0776-42c5-bc96-e36def08b74d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-27191095"}, "namespace": "ai-tenant-e2e-mt-27191095", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3ddedcd0-0776-42c5-bc96-e36def08b74d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-27191095"}
2026-07-10T15:17:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "138500ca-c855-47e2-b22f-9696c6665406", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-trusted-ca-bundle", "reconcileID": "9a8ce853-240d-47dc-84e5-8f90336bbcb8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:27Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "28cac170-700d-4816-aedd-d56d6e9361a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:27Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "28cac170-700d-4816-aedd-d56d6e9361a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:27Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "8311c609-f46a-4963-a89c-8e17fe3a26eb", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:27Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "8311c609-f46a-4963-a89c-8e17fe3a26eb", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:17:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fec7926e-9205-4934-b789-515c299f3459", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "2d52f69d-eab7-4031-97d1-69659c7c4e23", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:28Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-trusted-ca-bundle", "reconcileID": "780bad88-0c4e-4807-a70f-199333860a04", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-14be3db1", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:28Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-trusted-ca-bundle", "reconcileID": "780bad88-0c4e-4807-a70f-199333860a04", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:28Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e4bfb8a8-dded-46eb-a756-510e059d07e6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7cc70a99-c508-4e1c-8d2d-92850b6e8a09", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7cc70a99-c508-4e1c-8d2d-92850b6e8a09", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-14be3db1", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:48Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7cc70a99-c508-4e1c-8d2d-92850b6e8a09", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-14be3db1 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:17:48Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-trusted-ca-bundle", "reconcileID": "fe34181e-fe6b-429e-81ac-24dec85f9c9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-trusted-ca-bundle", "reconcileID": "fe34181e-fe6b-429e-81ac-24dec85f9c9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "1239e95b-129d-4c58-a4d8-c8ea5912fca6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "openshift-service-ca.crt", "reconcileID": "1239e95b-129d-4c58-a4d8-c8ea5912fca6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "786e6984-3172-4118-91fe-2ed59392fad8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:48Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-14be3db1"}, "namespace": "e2e-unlabeled-14be3db1", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "786e6984-3172-4118-91fe-2ed59392fad8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-14be3db1"}
2026-07-10T15:17:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "3b1c03ca-2743-4390-99b7-e0fa85e59442", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:17:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-trusted-ca-bundle", "reconcileID": "4f8ff1fa-30e9-41cc-83d2-da28ca2ea77b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:17:49Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "933df409-afb5-4376-bb23-cf800d43f80a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:49Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "933df409-afb5-4376-bb23-cf800d43f80a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:17:49Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "bf7af173-cfa5-4b8d-bfc6-5f603640fec5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:49Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "bf7af173-cfa5-4b8d-bfc6-5f603640fec5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:17:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a855df5-7618-41a3-a4c1-0a07a2c5fc47", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:17:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "9a657d9f-0310-44c2-99d4-a7d81a2d14ea", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:17:49Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-trusted-ca-bundle", "reconcileID": "718bafbf-5984-49d7-83ee-8f97acd9a620", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:17:49Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-trusted-ca-bundle", "reconcileID": "718bafbf-5984-49d7-83ee-8f97acd9a620", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:17:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a4e8ada4-9b6f-4159-9645-e66c20863c8c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d93e9450-50d3-4c69-8997-4028234208d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d93e9450-50d3-4c69-8997-4028234208d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:07Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d93e9450-50d3-4c69-8997-4028234208d6", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-a44aa4e2 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:18:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-trusted-ca-bundle", "reconcileID": "0edcf5ed-918c-4235-bc2d-50e108f274e1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-trusted-ca-bundle", "reconcileID": "0edcf5ed-918c-4235-bc2d-50e108f274e1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "baa609ae-ccbf-403b-af2c-759a81f4aad7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "openshift-service-ca.crt", "reconcileID": "baa609ae-ccbf-403b-af2c-759a81f4aad7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b74cbe4c-0330-4b58-8ce2-e5ee64398dce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a44aa4e2"}, "namespace": "ai-tenant-e2e-mt-a44aa4e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b74cbe4c-0330-4b58-8ce2-e5ee64398dce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a44aa4e2"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "openshift-service-ca.crt", "reconcileID": "d0b4d7fc-2541-40c5-b9c2-16f56da9f9d8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-trusted-ca-bundle", "reconcileID": "ab65be58-2100-4b54-9aa7-ccea9fcdef9f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "openshift-service-ca.crt", "reconcileID": "bc263be4-b9b4-4ef5-ba9f-898e447389cf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b25cf9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:07Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "openshift-service-ca.crt", "reconcileID": "bc263be4-b9b4-4ef5-ba9f-898e447389cf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4717dd48-e1af-4b8a-844d-128cd5399b03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "openshift-service-ca.crt", "reconcileID": "cfb330db-01ed-4abf-a969-77422711ad5f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-trusted-ca-bundle", "reconcileID": "096df05d-35f2-4ace-8eaf-7fae54472b5d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:07Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-trusted-ca-bundle", "reconcileID": "096df05d-35f2-4ace-8eaf-7fae54472b5d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0dcd714-bb4a-408f-b270-01f660f87637", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "openshift-service-ca.crt", "reconcileID": "15311f46-fe48-446b-b036-ee85c77c64a2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-trusted-ca-bundle", "reconcileID": "d9b714e3-ba97-45e3-8996-c2eeb02a3bfd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:08Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "openshift-service-ca.crt", "reconcileID": "6b380fca-974d-4323-b6e4-fe08e745e294", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e0eff3ea", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:08Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "openshift-service-ca.crt", "reconcileID": "6b380fca-974d-4323-b6e4-fe08e745e294", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1707cc3f-fbb8-4c0b-ac9b-2a85b35e92a7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "openshift-service-ca.crt", "reconcileID": "041b9427-c9c8-4648-b8b6-2f13fafc3ab0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:08Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-trusted-ca-bundle", "reconcileID": "7faecbca-77e5-44fc-9267-ce0fc1fb5fa6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:08Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-trusted-ca-bundle", "reconcileID": "7faecbca-77e5-44fc-9267-ce0fc1fb5fa6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d815d1d9-fcfa-41ac-93c2-daa3530e1c3e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:16Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "46734f3e-f9a8-41a5-ba19-1362842bcb36", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:16Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "46734f3e-f9a8-41a5-ba19-1362842bcb36", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:16Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "46734f3e-f9a8-41a5-ba19-1362842bcb36", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-75b25cf9 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:18:16Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-trusted-ca-bundle", "reconcileID": "fe2cc94a-e5e2-4a6e-9a8e-d7205af3da0f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-trusted-ca-bundle", "reconcileID": "fe2cc94a-e5e2-4a6e-9a8e-d7205af3da0f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:16Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "openshift-service-ca.crt", "reconcileID": "a5b0df38-2053-4706-a162-aaa7eea32c96", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "openshift-service-ca.crt", "reconcileID": "a5b0df38-2053-4706-a162-aaa7eea32c96", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:16Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "25f63455-55d4-453c-be9a-07da0b14ffdd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b25cf9"}, "namespace": "ai-tenant-e2e-mt-75b25cf9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "25f63455-55d4-453c-be9a-07da0b14ffdd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b25cf9"}
2026-07-10T15:18:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f8adb73b-85db-45f7-93b9-dac718c1313e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:22Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f8adb73b-85db-45f7-93b9-dac718c1313e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:22Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f8adb73b-85db-45f7-93b9-dac718c1313e", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-e0eff3ea because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:18:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-trusted-ca-bundle", "reconcileID": "ffe7173b-191a-4558-ab2d-8d4cf5cd5fb4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:22Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-trusted-ca-bundle", "reconcileID": "ffe7173b-191a-4558-ab2d-8d4cf5cd5fb4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "openshift-service-ca.crt", "reconcileID": "7232261d-e756-4461-bfae-3c5ae7d50fdb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:22Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "openshift-service-ca.crt", "reconcileID": "7232261d-e756-4461-bfae-3c5ae7d50fdb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d9a1c0db-e86f-414b-8b6e-e1f6235bcbbc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:22Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e0eff3ea"}, "namespace": "ai-tenant-e2e-mt-e0eff3ea", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d9a1c0db-e86f-414b-8b6e-e1f6235bcbbc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e0eff3ea"}
2026-07-10T15:18:24Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "56c8c34e-dd97-49ba-8802-955b71db018f", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "56c8c34e-dd97-49ba-8802-955b71db018f", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress", "name": "e2e-mt-867db321-authn-ssl"}
2026-07-10T15:18:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "56c8c34e-dd97-49ba-8802-955b71db018f", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "cd43c76f-a8ee-4952-ac4c-62ac3a455b0c", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "cd43c76f-a8ee-4952-ac4c-62ac3a455b0c", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress", "name": "e2e-mt-867db321-authn-ssl"}
2026-07-10T15:18:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "cd43c76f-a8ee-4952-ac4c-62ac3a455b0c", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "9db28868-3677-4b77-8e35-b957038ffd17", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "9db28868-3677-4b77-8e35-b957038ffd17", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress", "name": "e2e-mt-867db321-authn-ssl"}
2026-07-10T15:18:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "9db28868-3677-4b77-8e35-b957038ffd17", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "682297b0-0028-4bdc-a2e3-ac2abd273296", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "682297b0-0028-4bdc-a2e3-ac2abd273296", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress", "name": "e2e-mt-867db321-authn-ssl"}
2026-07-10T15:18:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "682297b0-0028-4bdc-a2e3-ac2abd273296", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "e0d6e3bb-1561-49de-ba18-26c89478722d", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "e0d6e3bb-1561-49de-ba18-26c89478722d", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress", "name": "e2e-mt-867db321-authn-ssl"}
2026-07-10T15:18:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "e0d6e3bb-1561-49de-ba18-26c89478722d", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "ff682646-b6a1-4a29-9f51-ec266ab9df8b", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:24Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "ff682646-b6a1-4a29-9f51-ec266ab9df8b", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress", "name": "e2e-mt-867db321-authn-ssl"}
2026-07-10T15:18:24Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "ff682646-b6a1-4a29-9f51-ec266ab9df8b", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:18:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "openshift-service-ca.crt", "reconcileID": "36885126-ff7e-46b9-b69e-ea9fba5cb182", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:18:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-trusted-ca-bundle", "reconcileID": "a00cbaff-7de7-461a-8a12-aa684cdc9b49", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:18:27Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "openshift-service-ca.crt", "reconcileID": "4176647f-dd45-4253-a453-63f05f53267c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-867db321", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:27Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "openshift-service-ca.crt", "reconcileID": "4176647f-dd45-4253-a453-63f05f53267c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:18:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "openshift-service-ca.crt", "reconcileID": "75183530-c802-4421-b6d6-45d33527f787", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:18:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5a65f92d-64a8-4c05-9248-582422c59f57", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:18:27Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-trusted-ca-bundle", "reconcileID": "d9fd6d74-cb96-45c5-ab57-b37bf0bd37fd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:18:27Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-trusted-ca-bundle", "reconcileID": "d9fd6d74-cb96-45c5-ab57-b37bf0bd37fd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:18:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "75f9dd94-01eb-4739-8e50-3e3c2c7984b7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e80acfb0-aac4-4f15-b814-938dc194b261", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e80acfb0-aac4-4f15-b814-938dc194b261", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:42Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e80acfb0-aac4-4f15-b814-938dc194b261", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-867db321 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:21:42Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-trusted-ca-bundle", "reconcileID": "a610aa34-5cfc-4e41-a71f-4958b9426e20", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-trusted-ca-bundle", "reconcileID": "a610aa34-5cfc-4e41-a71f-4958b9426e20", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "openshift-service-ca.crt", "reconcileID": "9159af64-566c-4036-a10d-688efb1f5233", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "openshift-service-ca.crt", "reconcileID": "9159af64-566c-4036-a10d-688efb1f5233", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7496cadf-4ff6-4878-8583-2281397d7dfe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:42Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-867db321"}, "namespace": "ai-tenant-e2e-mt-867db321", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7496cadf-4ff6-4878-8583-2281397d7dfe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-867db321"}
2026-07-10T15:21:44Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-867db321","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-867db321", "reconcileID": "2d99727b-257a-4cc8-8486-ddf7279377a4", "Gateway": "e2e-mt-867db321", "namespace": "openshift-ingress"}
2026-07-10T15:21:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-trusted-ca-bundle", "reconcileID": "e045a284-f82b-47fe-9a2f-e91004ed4440", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "openshift-service-ca.crt", "reconcileID": "a3f66150-c20c-4ac6-8e80-077ffd61b099", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:44Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "openshift-service-ca.crt", "reconcileID": "ca6e98c3-8a18-4a2f-9389-a579a123513b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-bbdbfb93", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:44Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "openshift-service-ca.crt", "reconcileID": "ca6e98c3-8a18-4a2f-9389-a579a123513b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc8dadfd-6dea-408a-94ae-b39f6aa82607", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "openshift-service-ca.crt", "reconcileID": "9c8143e7-f123-4bce-a1d3-b38dac162fa7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:44Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-trusted-ca-bundle", "reconcileID": "1fc7a4cb-827b-47bb-b12b-529239e7e2a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:44Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-trusted-ca-bundle", "reconcileID": "1fc7a4cb-827b-47bb-b12b-529239e7e2a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "99f67254-d140-4a63-a49a-92867f3c1c36", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a97f509-3ac1-49b8-944d-ab1ab1aa5aea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a97f509-3ac1-49b8-944d-ab1ab1aa5aea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:51Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a97f509-3ac1-49b8-944d-ab1ab1aa5aea", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-bbdbfb93 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:21:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-trusted-ca-bundle", "reconcileID": "4880a86f-323c-4fd3-887e-379018bab909", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-trusted-ca-bundle", "reconcileID": "4880a86f-323c-4fd3-887e-379018bab909", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "openshift-service-ca.crt", "reconcileID": "41838d1d-5ab0-4627-81f4-ce2c0b045fb2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "openshift-service-ca.crt", "reconcileID": "41838d1d-5ab0-4627-81f4-ce2c0b045fb2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "410a58e0-153b-4eb6-a960-29cbbe99f207", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-bbdbfb93"}, "namespace": "e2e-webhook-sub-bbdbfb93", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "410a58e0-153b-4eb6-a960-29cbbe99f207", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-bbdbfb93"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "openshift-service-ca.crt", "reconcileID": "b93b76a4-5465-400e-83ff-0528e3d42782", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-trusted-ca-bundle", "reconcileID": "b62d385b-8bfd-402b-be9c-003dcbec784f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "openshift-service-ca.crt", "reconcileID": "5cac125b-dc30-4c73-91af-8ee9d01ae3db", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1039a80c", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:51Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "openshift-service-ca.crt", "reconcileID": "5cac125b-dc30-4c73-91af-8ee9d01ae3db", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "89580d30-070a-4ebd-9326-49dda27f59be", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "openshift-service-ca.crt", "reconcileID": "aa2e72d8-ec30-4710-b1ba-65987b3e9c4c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-trusted-ca-bundle", "reconcileID": "cf22c8a6-39f4-4d75-a7bd-c04df66072a5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:51Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-trusted-ca-bundle", "reconcileID": "cf22c8a6-39f4-4d75-a7bd-c04df66072a5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "00c8551f-af85-4a2e-94fa-871c376b8a5e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "146dcaab-8ae1-426a-9d0c-9907448aac89", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "146dcaab-8ae1-426a-9d0c-9907448aac89", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:21:58Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "146dcaab-8ae1-426a-9d0c-9907448aac89", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-1039a80c because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:21:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-trusted-ca-bundle", "reconcileID": "d875ac4c-7627-4ae7-b91a-196d5448102d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-trusted-ca-bundle", "reconcileID": "d875ac4c-7627-4ae7-b91a-196d5448102d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "openshift-service-ca.crt", "reconcileID": "f70b5b8c-76db-4d04-9c82-8c15643115a1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "openshift-service-ca.crt", "reconcileID": "f70b5b8c-76db-4d04-9c82-8c15643115a1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f18e8d6-d1c7-4fa3-b6eb-1aef3c790fda", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:21:58Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-1039a80c"}, "namespace": "e2e-webhook-auth-1039a80c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f18e8d6-d1c7-4fa3-b6eb-1aef3c790fda", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-1039a80c"}
2026-07-10T15:22:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "62ad9593-f577-4662-bb0e-d8ab8a60d2c4", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "62ad9593-f577-4662-bb0e-d8ab8a60d2c4", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "62ad9593-f577-4662-bb0e-d8ab8a60d2c4", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "6e8866e9-e0bc-4712-8152-74c5e93b4ad0", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "6e8866e9-e0bc-4712-8152-74c5e93b4ad0", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "6e8866e9-e0bc-4712-8152-74c5e93b4ad0", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "7cf3e4fa-de9f-4ea9-891a-00729da9813d", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "7cf3e4fa-de9f-4ea9-891a-00729da9813d", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "7cf3e4fa-de9f-4ea9-891a-00729da9813d", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "7a60453e-5bfe-4e63-bb71-33f8171ed5e4", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "7a60453e-5bfe-4e63-bb71-33f8171ed5e4", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "7a60453e-5bfe-4e63-bb71-33f8171ed5e4", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "4eafd701-822d-44e7-a419-c735c38d3461", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "4eafd701-822d-44e7-a419-c735c38d3461", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "4eafd701-822d-44e7-a419-c735c38d3461", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "ffa24eeb-3b17-425a-a18c-41db5741c9fe", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "ffa24eeb-3b17-425a-a18c-41db5741c9fe", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "ffa24eeb-3b17-425a-a18c-41db5741c9fe", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:06Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "d498a62b-8eb9-407e-8921-8fff185cbad0", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:06Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "d498a62b-8eb9-407e-8921-8fff185cbad0", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f-authn-ssl"}
2026-07-10T15:22:06Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "d498a62b-8eb9-407e-8921-8fff185cbad0", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:22:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "openshift-service-ca.crt", "reconcileID": "aaf8d52a-29bc-4a19-b174-d1176327279c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:22:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-trusted-ca-bundle", "reconcileID": "bb81097d-e59a-4a73-98e0-166428ec6b9f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:22:08Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "openshift-service-ca.crt", "reconcileID": "8603907b-5a9a-4fc0-83b0-565298f05ecf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c21f337f", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:22:08Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "openshift-service-ca.crt", "reconcileID": "8603907b-5a9a-4fc0-83b0-565298f05ecf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:22:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0fd3a3c8-6d69-47eb-bf27-b0eba1d9a562", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:22:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "openshift-service-ca.crt", "reconcileID": "8ce83a34-26a9-41db-948b-8ae1988e3043", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:22:08Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-trusted-ca-bundle", "reconcileID": "749da1d6-f202-41e1-b978-7c1806b790a4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:22:08Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-trusted-ca-bundle", "reconcileID": "749da1d6-f202-41e1-b978-7c1806b790a4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:22:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a23d777f-0bfe-418d-b395-a91c653f7856", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8da6ed7-2a06-4f56-ad00-fe7006435c1f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8da6ed7-2a06-4f56-ad00-fe7006435c1f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:23Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8da6ed7-2a06-4f56-ad00-fe7006435c1f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c21f337f because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:25:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-trusted-ca-bundle", "reconcileID": "66b6ec65-c4d6-461f-8f94-b49fdb0f697f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-trusted-ca-bundle", "reconcileID": "66b6ec65-c4d6-461f-8f94-b49fdb0f697f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "openshift-service-ca.crt", "reconcileID": "260a228d-257a-4c88-b1bf-bf172c11c703", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "openshift-service-ca.crt", "reconcileID": "260a228d-257a-4c88-b1bf-bf172c11c703", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f0cf7e7e-06aa-4652-a3e8-3d92ceebc609", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c21f337f"}, "namespace": "ai-tenant-e2e-mt-c21f337f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f0cf7e7e-06aa-4652-a3e8-3d92ceebc609", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c21f337f"}
2026-07-10T15:25:24Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c21f337f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c21f337f", "reconcileID": "02bcbb7e-a075-473b-a5f2-a37d45fd1a8d", "Gateway": "e2e-mt-c21f337f", "namespace": "openshift-ingress"}
2026-07-10T15:25:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "openshift-service-ca.crt", "reconcileID": "2935568e-5b0d-41c4-8726-e90c50b15107", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-trusted-ca-bundle", "reconcileID": "31f97c11-0c64-43b8-8c27-e239a9dc774d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:25Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "openshift-service-ca.crt", "reconcileID": "f749c308-0aaa-4440-bddf-548b332e7bd2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c72e2fb8", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:25Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "openshift-service-ca.crt", "reconcileID": "f749c308-0aaa-4440-bddf-548b332e7bd2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "94ec400f-9276-4172-819e-38d483d59b7d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "openshift-service-ca.crt", "reconcileID": "82510697-ce21-42e6-a2c0-ec5178de41a1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:25Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-trusted-ca-bundle", "reconcileID": "cd463dd8-8b54-43cf-903b-1b59bb290359", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:25Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-trusted-ca-bundle", "reconcileID": "cd463dd8-8b54-43cf-903b-1b59bb290359", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b139c2e0-59c6-48bf-bbc9-d3bbb2db07a0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "openshift-service-ca.crt", "reconcileID": "54edfd07-e0e7-4a3f-85e4-59af731a8c29", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-trusted-ca-bundle", "reconcileID": "759da8c4-2c09-40ab-a363-167d3711f3cb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:26Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "openshift-service-ca.crt", "reconcileID": "6d0574fd-4155-4066-85f4-059ce2ea0d85", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-49697e74", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:26Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "openshift-service-ca.crt", "reconcileID": "6d0574fd-4155-4066-85f4-059ce2ea0d85", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d1ee6ef7-6318-4a87-b9ec-c6842ca76e35", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "openshift-service-ca.crt", "reconcileID": "a01ffe28-e161-44ad-adf9-6a77a0623c8d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:26Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-trusted-ca-bundle", "reconcileID": "8878f6c9-e4ba-4700-927c-9074ea9b4e8d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:26Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-trusted-ca-bundle", "reconcileID": "8878f6c9-e4ba-4700-927c-9074ea9b4e8d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a956986-0a68-4c4b-9bda-539e80869172", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f45fa982-4798-4943-9e78-f03e8d61f5e7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f45fa982-4798-4943-9e78-f03e8d61f5e7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:33Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f45fa982-4798-4943-9e78-f03e8d61f5e7", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c72e2fb8 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:25:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-trusted-ca-bundle", "reconcileID": "9f2fb0bd-c730-404d-82bc-36fcfc5823e8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-trusted-ca-bundle", "reconcileID": "9f2fb0bd-c730-404d-82bc-36fcfc5823e8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "openshift-service-ca.crt", "reconcileID": "48bf9eff-a065-44e3-835b-67219f4452d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "openshift-service-ca.crt", "reconcileID": "48bf9eff-a065-44e3-835b-67219f4452d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c5529cc-9e14-42ed-a679-3b70912f8308", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c72e2fb8"}, "namespace": "ai-tenant-e2e-mt-c72e2fb8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c5529cc-9e14-42ed-a679-3b70912f8308", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c72e2fb8"}
2026-07-10T15:25:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1c15cce8-087a-46a2-acdf-ce1fbe9f7e1a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1c15cce8-087a-46a2-acdf-ce1fbe9f7e1a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:40Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1c15cce8-087a-46a2-acdf-ce1fbe9f7e1a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-49697e74 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:25:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-trusted-ca-bundle", "reconcileID": "0eb52a87-4290-4df6-b058-e5aca6ede534", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-trusted-ca-bundle", "reconcileID": "0eb52a87-4290-4df6-b058-e5aca6ede534", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "openshift-service-ca.crt", "reconcileID": "bbc3573e-8810-4906-a7a9-40539e24cac4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "openshift-service-ca.crt", "reconcileID": "bbc3573e-8810-4906-a7a9-40539e24cac4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "330bcf50-617f-4a64-89b2-ccd63ea20085", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-49697e74"}, "namespace": "ai-tenant-e2e-mt-49697e74", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "330bcf50-617f-4a64-89b2-ccd63ea20085", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-49697e74"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "openshift-service-ca.crt", "reconcileID": "ffa70e70-cedd-476b-bd52-6c3c5b7a17d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-trusted-ca-bundle", "reconcileID": "53ae4e73-42f6-45bc-80d4-929e413dd854", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:25:40Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "openshift-service-ca.crt", "reconcileID": "c74a6fe6-7fec-49b1-99c5-44356d3a7a2e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-670b9d13", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:40Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "openshift-service-ca.crt", "reconcileID": "c74a6fe6-7fec-49b1-99c5-44356d3a7a2e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f77ea9f8-975f-4439-bd91-ae9b742f2ff6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "openshift-service-ca.crt", "reconcileID": "2aa7d6a8-ba94-4e32-a015-f55f03ce4f17", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:25:40Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-trusted-ca-bundle", "reconcileID": "eed44b1d-b227-4300-8369-27e55ba88701", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:25:40Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-trusted-ca-bundle", "reconcileID": "eed44b1d-b227-4300-8369-27e55ba88701", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:25:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f5417b0b-2d19-48bb-8242-21bf07736350", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3d5643bd-10d0-4567-b397-26f979b82ded", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3d5643bd-10d0-4567-b397-26f979b82ded", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:26:18Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3d5643bd-10d0-4567-b397-26f979b82ded", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-670b9d13 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:26:18Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-trusted-ca-bundle", "reconcileID": "a104c098-c672-4688-ae49-735d51a8b736", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-trusted-ca-bundle", "reconcileID": "a104c098-c672-4688-ae49-735d51a8b736", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "openshift-service-ca.crt", "reconcileID": "72b26ea4-8cfb-4ffa-a644-2841770f6623", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "openshift-service-ca.crt", "reconcileID": "72b26ea4-8cfb-4ffa-a644-2841770f6623", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a255ef14-2d59-4900-8ffb-cdc4e31434ef", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-670b9d13"}, "namespace": "ai-tenant-e2e-mt-670b9d13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a255ef14-2d59-4900-8ffb-cdc4e31434ef", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-670b9d13"}
2026-07-10T15:26:18Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "d40e0e5b-f30d-4f31-82c2-f6e17f8095ae", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "d40e0e5b-f30d-4f31-82c2-f6e17f8095ae", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:18Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "d40e0e5b-f30d-4f31-82c2-f6e17f8095ae", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "897b930b-a2d0-4e18-aabc-8f62a1f4314f", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "897b930b-a2d0-4e18-aabc-8f62a1f4314f", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:18Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "897b930b-a2d0-4e18-aabc-8f62a1f4314f", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "23e8b832-7f70-409b-b89e-84213b30e771", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "23e8b832-7f70-409b-b89e-84213b30e771", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:18Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "23e8b832-7f70-409b-b89e-84213b30e771", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "bbbbe865-0ed2-4e3b-b16f-8388b3f57127", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:18Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "bbbbe865-0ed2-4e3b-b16f-8388b3f57127", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:18Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "bbbbe865-0ed2-4e3b-b16f-8388b3f57127", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:19Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "0d119682-5b10-41d2-b06c-ae8a4510f572", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:19Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "0d119682-5b10-41d2-b06c-ae8a4510f572", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:19Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "0d119682-5b10-41d2-b06c-ae8a4510f572", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:19Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "0d87533f-9554-4897-9630-c2cb49d459b1", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:19Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "0d87533f-9554-4897-9630-c2cb49d459b1", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:19Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "0d87533f-9554-4897-9630-c2cb49d459b1", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "e74e85e1-30c9-4270-9f67-b25501b13a91", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:20Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "e74e85e1-30c9-4270-9f67-b25501b13a91", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18-authn-ssl"}
2026-07-10T15:26:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "e74e85e1-30c9-4270-9f67-b25501b13a91", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}
2026-07-10T15:26:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "openshift-service-ca.crt", "reconcileID": "678845b8-f6ae-4bd5-b538-35b60baf6cfd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:26:21Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "openshift-service-ca.crt", "reconcileID": "52fadd5b-856c-4106-8c29-b3ae0059321c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-566b18", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:26:21Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "openshift-service-ca.crt", "reconcileID": "52fadd5b-856c-4106-8c29-b3ae0059321c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:26:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-trusted-ca-bundle", "reconcileID": "2caeefed-c42d-40df-9115-113a1024c718", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:26:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "feb7209e-cb91-43a4-b8e5-df3357cfc561", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:26:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "openshift-service-ca.crt", "reconcileID": "4142caf1-36d4-495c-a23e-a68ecb906348", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:26:21Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-trusted-ca-bundle", "reconcileID": "fffbeffc-0132-4806-b05e-93d9ee25300d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:26:21Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-trusted-ca-bundle", "reconcileID": "fffbeffc-0132-4806-b05e-93d9ee25300d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:26:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b9f4034b-4c1c-435a-899e-4d465ece8dfe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c976105-039c-43bc-b8e2-95eaa2d77972", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c976105-039c-43bc-b8e2-95eaa2d77972", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T15:29:37Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c976105-039c-43bc-b8e2-95eaa2d77972", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-566b18 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T15:29:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-trusted-ca-bundle", "reconcileID": "d9669e0f-cc58-48db-995c-fabf21e52a64", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-trusted-ca-bundle", "reconcileID": "d9669e0f-cc58-48db-995c-fabf21e52a64", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "openshift-service-ca.crt", "reconcileID": "64efb619-fedf-4fd9-a4e8-1ac56fa30263", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "openshift-service-ca.crt", "reconcileID": "64efb619-fedf-4fd9-a4e8-1ac56fa30263", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9cb9a2b-7ed5-4f94-ba3b-d7e49da00e8e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-566b18"}, "namespace": "ai-tenant-e2e-inf-a-566b18", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9cb9a2b-7ed5-4f94-ba3b-d7e49da00e8e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-566b18"}
2026-07-10T15:29:37Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-566b18","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-566b18", "reconcileID": "a4dba5f7-7b6c-43a5-882f-1033175b6b46", "Gateway": "e2e-inf-a-566b18", "namespace": "openshift-ingress"}