2026-07-07T19:45:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-d70e5b04"}, "namespace": "ai-tenant-e2e-ait-d70e5b04", "name": "odh-trusted-ca-bundle", "reconcileID": "e87fb2c2-6408-4b8e-9083-fbe1a9349333", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-d70e5b04"} 2026-07-07T19:45:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-d70e5b04"}, "namespace": "ai-tenant-e2e-ait-d70e5b04", "name": "odh-trusted-ca-bundle", "reconcileID": "e87fb2c2-6408-4b8e-9083-fbe1a9349333", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-d70e5b04"} 2026-07-07T19:45:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-d70e5b04"}, "namespace": "ai-tenant-e2e-ait-d70e5b04", "name": "openshift-service-ca.crt", "reconcileID": "98ef9da0-41c9-414d-b0fd-aa5096f15b1a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-d70e5b04"} 2026-07-07T19:45:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-d70e5b04"}, "namespace": "ai-tenant-e2e-ait-d70e5b04", "name": "openshift-service-ca.crt", "reconcileID": "98ef9da0-41c9-414d-b0fd-aa5096f15b1a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-d70e5b04"} 2026-07-07T19:45:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-d70e5b04"}, "namespace": "ai-tenant-e2e-ait-d70e5b04", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "602cb99e-9ff0-49de-a0da-13e49cea26d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-d70e5b04"} 2026-07-07T19:45:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-d70e5b04"}, "namespace": "ai-tenant-e2e-ait-d70e5b04", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "602cb99e-9ff0-49de-a0da-13e49cea26d6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-d70e5b04"} 2026-07-07T19:45:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "f7dcf3df-d6f8-431d-9f9d-eed75a9e3d20", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:45:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "a897dd24-85dc-46bb-a1fb-be74203584de", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:45:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "77000e68-c006-4198-8615-75655e96aba7", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:45:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "0c3b269d-ef9c-4e09-85fa-1d305531bfe2", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:45:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "bc0ab2c7-3531-4cb8-baf0-35163c3365ea", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:45:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "4ce267bb-fa71-42c8-806a-170e9d3bb28f", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:45:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "openshift-service-ca.crt", "reconcileID": "ec89dccc-c991-40bb-8408-0cc5ddc13c30", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-trusted-ca-bundle", "reconcileID": "d5524256-96b6-4a0f-a22f-7c4ba3d19841", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "openshift-service-ca.crt", "reconcileID": "2306bc53-1dbb-42e1-88c0-afda3dae0f01", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-2b04b691", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:45:57Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "openshift-service-ca.crt", "reconcileID": "2306bc53-1dbb-42e1-88c0-afda3dae0f01", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74f9a8ce-b530-4363-ae1f-46c392190347", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "openshift-service-ca.crt", "reconcileID": "72e0a3c7-448b-46e3-9e42-82297b4e16f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-trusted-ca-bundle", "reconcileID": "9458833d-9dc1-4470-a9fc-f1994e2d9315", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:45:57Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-trusted-ca-bundle", "reconcileID": "9458833d-9dc1-4470-a9fc-f1994e2d9315", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "38254588-fbc4-4d2c-8a2e-844c3a078175", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:45:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "bc98237d-4fc8-4e3c-b37e-4f91378f3b46", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:45:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-2b04b691","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-2b04b691", "reconcileID": "d15c9440-2402-4dc5-8505-5fa5f19befcd", "Gateway": "e2e-derive-2b04b691", "namespace": "openshift-ingress"} 2026-07-07T19:46:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6bc2073-4988-4d3c-afc3-2754c91bd078", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6bc2073-4988-4d3c-afc3-2754c91bd078", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:08Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6bc2073-4988-4d3c-afc3-2754c91bd078", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-derive-2b04b691 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:46:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-trusted-ca-bundle", "reconcileID": "c498f8e0-6c9b-4be0-a801-056480b20acc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-trusted-ca-bundle", "reconcileID": "c498f8e0-6c9b-4be0-a801-056480b20acc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "openshift-service-ca.crt", "reconcileID": "115f54cd-b3de-4273-81e5-19fcfb903878", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "openshift-service-ca.crt", "reconcileID": "115f54cd-b3de-4273-81e5-19fcfb903878", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6cc42a8a-1e0d-4117-bb82-32d36901ee2c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-2b04b691"}, "namespace": "ai-tenant-e2e-derive-2b04b691", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6cc42a8a-1e0d-4117-bb82-32d36901ee2c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-2b04b691"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "openshift-service-ca.crt", "reconcileID": "0a826504-e1d4-4523-b06a-00ef609e2b7f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-trusted-ca-bundle", "reconcileID": "9b79c4e1-7641-4d40-94ca-c1453537b2b1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "openshift-service-ca.crt", "reconcileID": "fa10a85f-0352-46bd-b3ab-0707f2ed7100", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2f60fa82", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:08Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "openshift-service-ca.crt", "reconcileID": "fa10a85f-0352-46bd-b3ab-0707f2ed7100", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d863b430-a85f-4428-ada2-7d35947ecdb2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "openshift-service-ca.crt", "reconcileID": "81dfb050-e79c-4108-b31b-facec68b730c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-trusted-ca-bundle", "reconcileID": "91e9ee7a-6a38-4f78-8d3f-5e87e9ee2581", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:08Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-trusted-ca-bundle", "reconcileID": "91e9ee7a-6a38-4f78-8d3f-5e87e9ee2581", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ec5b77cd-41e5-4a72-b0e4-86547506ac31", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e55b6b6-3d97-4c4a-aac2-c22eb58cf0db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e55b6b6-3d97-4c4a-aac2-c22eb58cf0db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e55b6b6-3d97-4c4a-aac2-c22eb58cf0db", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-2f60fa82 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:46:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-trusted-ca-bundle", "reconcileID": "59c1a11a-5d4f-48df-a9fd-393fe55171fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-trusted-ca-bundle", "reconcileID": "59c1a11a-5d4f-48df-a9fd-393fe55171fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "openshift-service-ca.crt", "reconcileID": "3008a320-19d7-4467-841d-2523df24efb2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "openshift-service-ca.crt", "reconcileID": "3008a320-19d7-4467-841d-2523df24efb2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c06d8125-6566-4965-9b7b-72bd61f93c04", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-2f60fa82"}, "namespace": "ai-tenant-e2e-mt-2f60fa82", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c06d8125-6566-4965-9b7b-72bd61f93c04", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-2f60fa82"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "openshift-service-ca.crt", "reconcileID": "6dbae60e-a216-4753-ace6-7e63c8168388", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-trusted-ca-bundle", "reconcileID": "31928e42-82b8-4af6-aeb7-87ecfec4e67b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "openshift-service-ca.crt", "reconcileID": "74698979-31b2-4977-96b3-81935e009dff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-05ad7e47", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:16Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "openshift-service-ca.crt", "reconcileID": "74698979-31b2-4977-96b3-81935e009dff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d1cf0494-8deb-4fdb-b740-b1c306fd686f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "openshift-service-ca.crt", "reconcileID": "9ba61397-64c1-4cde-8076-6d7006afa208", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-trusted-ca-bundle", "reconcileID": "b290b03b-685a-4e7a-9749-89d66b99fa7c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:16Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-trusted-ca-bundle", "reconcileID": "b290b03b-685a-4e7a-9749-89d66b99fa7c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e84bbed-0f3a-45a0-b28d-f7985c75dbc4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "53331c4f-2eea-4f94-9bc4-d8544389ccd8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "53331c4f-2eea-4f94-9bc4-d8544389ccd8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:48Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "53331c4f-2eea-4f94-9bc4-d8544389ccd8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-05ad7e47 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:46:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-trusted-ca-bundle", "reconcileID": "dd875159-e559-43cb-bcd3-bb38f60d8f71", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-trusted-ca-bundle", "reconcileID": "dd875159-e559-43cb-bcd3-bb38f60d8f71", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "openshift-service-ca.crt", "reconcileID": "2ead4d5d-d54f-48d5-bbd7-8227ef640b30", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "openshift-service-ca.crt", "reconcileID": "2ead4d5d-d54f-48d5-bbd7-8227ef640b30", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8ea5d81c-5e9b-4558-9afb-f7afad570d67", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-05ad7e47"}, "namespace": "ai-tenant-e2e-mt-05ad7e47", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8ea5d81c-5e9b-4558-9afb-f7afad570d67", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-05ad7e47"} 2026-07-07T19:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "openshift-service-ca.crt", "reconcileID": "895e347c-12f4-41d8-8896-6e547c61e66e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-trusted-ca-bundle", "reconcileID": "d446a38b-dfc5-4da1-9322-dd2922ece6e3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:46:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "openshift-service-ca.crt", "reconcileID": "03c6e430-6e9d-4f3d-ab20-6b51e0a6ee8c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-53b11397", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "openshift-service-ca.crt", "reconcileID": "03c6e430-6e9d-4f3d-ab20-6b51e0a6ee8c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5edcd5ed-593d-4288-ac3b-750879608561", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "openshift-service-ca.crt", "reconcileID": "676af21c-d73f-4d37-ab42-1ddb2f759ef0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:46:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-trusted-ca-bundle", "reconcileID": "75be75b5-3d70-49fb-a133-e109ab4ebbd5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-53b11397", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:46:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-trusted-ca-bundle", "reconcileID": "75be75b5-3d70-49fb-a133-e109ab4ebbd5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a6b75f5e-cfc8-4a5b-b444-d7b3a9ee86a8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6e81563-269d-4616-bb1b-23600928e654", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6e81563-269d-4616-bb1b-23600928e654", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-53b11397", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:11Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6e81563-269d-4616-bb1b-23600928e654", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-53b11397 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:47:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-trusted-ca-bundle", "reconcileID": "97685de9-2a26-4b4c-90d6-3ea8a5f764a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-trusted-ca-bundle", "reconcileID": "97685de9-2a26-4b4c-90d6-3ea8a5f764a1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "openshift-service-ca.crt", "reconcileID": "84bd5910-fa76-4649-b21c-e60d8f38472e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "openshift-service-ca.crt", "reconcileID": "84bd5910-fa76-4649-b21c-e60d8f38472e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8ea2a6e0-e69e-4f86-82f3-82bd9813affa", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-53b11397"}, "namespace": "e2e-unlabeled-53b11397", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8ea2a6e0-e69e-4f86-82f3-82bd9813affa", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-53b11397"} 2026-07-07T19:47:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-trusted-ca-bundle", "reconcileID": "ca54c36e-9ee6-42b3-8d47-8fbfa9f52c2e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "openshift-service-ca.crt", "reconcileID": "b21af29d-c37d-4c4b-9276-f6f7250bed00", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "openshift-service-ca.crt", "reconcileID": "2de42b2e-012a-4f24-998c-84f0356ce5d3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-25e54000", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "openshift-service-ca.crt", "reconcileID": "2de42b2e-012a-4f24-998c-84f0356ce5d3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "289b0300-90a4-4568-8291-258d65044ffe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "openshift-service-ca.crt", "reconcileID": "b2fcd234-cca0-4df0-afb4-6281e460d2ce", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-trusted-ca-bundle", "reconcileID": "5ed2dcbf-3f3b-465a-bc25-ba8ce7c2bc8e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-trusted-ca-bundle", "reconcileID": "5ed2dcbf-3f3b-465a-bc25-ba8ce7c2bc8e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "48da84fd-fb2a-4a04-8be8-f385ba4f8cc6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a6dc493-6fd8-4257-80a2-b3d60d77e0e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a6dc493-6fd8-4257-80a2-b3d60d77e0e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a6dc493-6fd8-4257-80a2-b3d60d77e0e8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-25e54000 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:47:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-trusted-ca-bundle", "reconcileID": "bad06015-1e1c-49f0-9420-90f58eeb8521", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-trusted-ca-bundle", "reconcileID": "bad06015-1e1c-49f0-9420-90f58eeb8521", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "openshift-service-ca.crt", "reconcileID": "eb624faa-2d37-4be3-9af8-e0b2911b1425", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "openshift-service-ca.crt", "reconcileID": "eb624faa-2d37-4be3-9af8-e0b2911b1425", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1cb46829-e8dc-4f16-8e84-270b6b87f1c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-25e54000"}, "namespace": "ai-tenant-e2e-mt-25e54000", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1cb46829-e8dc-4f16-8e84-270b6b87f1c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-25e54000"} 2026-07-07T19:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "e088b85f-f513-4cd8-93ef-df147056bc9d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-trusted-ca-bundle", "reconcileID": "cfccab8d-2246-4cf5-a50b-285a6447d12f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "f3f6747b-9e4c-4e07-bcc1-1f62697c7f92", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:30Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "f3f6747b-9e4c-4e07-bcc1-1f62697c7f92", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "a1728c26-3810-47b6-a6a5-be51345d901b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "a1728c26-3810-47b6-a6a5-be51345d901b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d96421ba-4b8d-47ed-ad5b-20db45424438", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "552fe38b-5bea-4461-bedd-8fa9b1d269f5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-trusted-ca-bundle", "reconcileID": "f86bbaf9-2769-4e31-b246-9e502379fa0c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:30Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-trusted-ca-bundle", "reconcileID": "f86bbaf9-2769-4e31-b246-9e502379fa0c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "234b044e-42d4-4faa-b18b-a4d6858fe365", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "openshift-service-ca.crt", "reconcileID": "b0b372b9-e3ea-49dd-801c-ca513d27d00f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "openshift-service-ca.crt", "reconcileID": "30778e4b-bcd6-4cd8-bbf2-d8db7c29b096", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-355030bf", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "openshift-service-ca.crt", "reconcileID": "30778e4b-bcd6-4cd8-bbf2-d8db7c29b096", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-trusted-ca-bundle", "reconcileID": "1b3a3876-38d9-42fd-ac37-db41ebf81f7b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "98a7d34e-87da-4701-beb1-459bbcd54ba5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "openshift-service-ca.crt", "reconcileID": "4ef9ce0e-621c-4f7d-a72b-8d2d2056609f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-trusted-ca-bundle", "reconcileID": "a932c84f-a99b-4024-98b7-77d3577f8d82", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-trusted-ca-bundle", "reconcileID": "a932c84f-a99b-4024-98b7-77d3577f8d82", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c78e8b5c-88b9-497e-905a-6ed6838cb621", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:38Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ab6ca8e2-887e-4d38-bee1-7c86e5a191d4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ab6ca8e2-887e-4d38-bee1-7c86e5a191d4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:38Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ab6ca8e2-887e-4d38-bee1-7c86e5a191d4", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-b3f131ff because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:47:38Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-trusted-ca-bundle", "reconcileID": "aa24eebb-b6b7-494b-97dc-ea1cc3d35e08", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-trusted-ca-bundle", "reconcileID": "aa24eebb-b6b7-494b-97dc-ea1cc3d35e08", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:38Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "fe2dcd4a-7ffc-4ea5-af47-b2b087c55875", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "openshift-service-ca.crt", "reconcileID": "fe2dcd4a-7ffc-4ea5-af47-b2b087c55875", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:38Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4006d7a8-71db-4b4f-90f1-463a7aae3074", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b3f131ff"}, "namespace": "ai-tenant-e2e-mt-b3f131ff", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4006d7a8-71db-4b4f-90f1-463a7aae3074", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b3f131ff"} 2026-07-07T19:47:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8012231c-2d3a-4d80-be68-fa13c202b3f0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8012231c-2d3a-4d80-be68-fa13c202b3f0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:45Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8012231c-2d3a-4d80-be68-fa13c202b3f0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-355030bf because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:47:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-trusted-ca-bundle", "reconcileID": "88089bbe-6f9c-4249-9e3f-040cd61bd373", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-trusted-ca-bundle", "reconcileID": "88089bbe-6f9c-4249-9e3f-040cd61bd373", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "openshift-service-ca.crt", "reconcileID": "f1b42747-96f7-4aab-bc29-28d309778e60", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "openshift-service-ca.crt", "reconcileID": "f1b42747-96f7-4aab-bc29-28d309778e60", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c4242e3-f107-493d-9c2c-85726864d4b4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-355030bf"}, "namespace": "ai-tenant-e2e-mt-355030bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c4242e3-f107-493d-9c2c-85726864d4b4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-355030bf"} 2026-07-07T19:47:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "5a1b435b-0977-4ce0-8f40-991e20f14b79", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "5a1b435b-0977-4ce0-8f40-991e20f14b79", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "5a1b435b-0977-4ce0-8f40-991e20f14b79", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "9b27048a-0631-4619-a414-5dc59ac7682e", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "9b27048a-0631-4619-a414-5dc59ac7682e", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "9b27048a-0631-4619-a414-5dc59ac7682e", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "e761aba6-d7ac-4fe9-b192-1f9112cd5b2d", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "e761aba6-d7ac-4fe9-b192-1f9112cd5b2d", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "e761aba6-d7ac-4fe9-b192-1f9112cd5b2d", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "d99907f2-6143-49de-8d13-e27850d89de4", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "d99907f2-6143-49de-8d13-e27850d89de4", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "d99907f2-6143-49de-8d13-e27850d89de4", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:47Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "c604ef58-bb84-4431-8edf-327c3d1786ca", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:47Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "c604ef58-bb84-4431-8edf-327c3d1786ca", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:47Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "c604ef58-bb84-4431-8edf-327c3d1786ca", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:47Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "dffb36de-8436-42fb-bc64-aa13ed6a96ce", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:47Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "dffb36de-8436-42fb-bc64-aa13ed6a96ce", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:47Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "dffb36de-8436-42fb-bc64-aa13ed6a96ce", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:48Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "8a4fb5f4-1f4e-4600-8496-43eb14125da5", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:48Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "8a4fb5f4-1f4e-4600-8496-43eb14125da5", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:48Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "8a4fb5f4-1f4e-4600-8496-43eb14125da5", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "openshift-service-ca.crt", "reconcileID": "e6a8be95-fe9f-40a1-b669-fe125ab2f8c1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "openshift-service-ca.crt", "reconcileID": "1ebe6976-7cc9-48d3-ace5-0147c8a13321", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a6d4b797", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:50Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "openshift-service-ca.crt", "reconcileID": "1ebe6976-7cc9-48d3-ace5-0147c8a13321", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "e365b4a4-2273-4407-9e0d-3792800e9baf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:50Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "e365b4a4-2273-4407-9e0d-3792800e9baf", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:47:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "openshift-service-ca.crt", "reconcileID": "eae7d534-5cb5-4d7e-8618-f59ca7c57f06", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7f57d10e-96ee-4d65-8395-7a39d22c7332", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "c7667118-b6b0-4806-8e88-c013c27deaea", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "c90c5489-03a7-41fc-b5b0-5775732b3d7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:47:50Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "c90c5489-03a7-41fc-b5b0-5775732b3d7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "70e84e59-8f88-4809-a1a4-54e467bcfbe8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:47:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "dfaddffb-9933-4beb-8233-a02ef9a0c21d", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "dfaddffb-9933-4beb-8233-a02ef9a0c21d", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "dfaddffb-9933-4beb-8233-a02ef9a0c21d", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "f9d959c2-5cb3-453b-ba86-f3e983caec59", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:47:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "397267b6-ad51-4761-b700-f54a301e7df0", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:47:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "397267b6-ad51-4761-b700-f54a301e7df0", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:47:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "397267b6-ad51-4761-b700-f54a301e7df0", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:48:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "138b6472-fc62-4aad-b8f6-242ee094a2f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:48:49Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:48:49Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:48:51Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:48:51Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} E0707 19:49:44.031462 1 leaderelection.go:441] Failed to update lock optimistically: Put "https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/odh-model-controller.opendatahub.io?timeout=5s": net/http: request canceled (Client.Timeout exceeded while awaiting headers), falling back to slow path 2026-07-07T19:50:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "b6adb89f-7f1b-4167-8492-f5579dfde432", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:50:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "965645a2-a7ed-44b8-91a6-8d7dfcfe3637", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:50:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "965645a2-a7ed-44b8-91a6-8d7dfcfe3637", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797-authn-ssl"} 2026-07-07T19:50:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a6d4b797","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a6d4b797", "reconcileID": "965645a2-a7ed-44b8-91a6-8d7dfcfe3637", "Gateway": "e2e-mt-a6d4b797", "namespace": "openshift-ingress"} 2026-07-07T19:51:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "07875dd6-5636-4f99-911e-f4b1647889bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "07875dd6-5636-4f99-911e-f4b1647889bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:03Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "07875dd6-5636-4f99-911e-f4b1647889bb", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-a6d4b797 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:51:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "72652b0c-03c5-4d74-9647-5d7c033ea5e9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-trusted-ca-bundle", "reconcileID": "72652b0c-03c5-4d74-9647-5d7c033ea5e9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "openshift-service-ca.crt", "reconcileID": "5e32860f-b949-4375-a2ef-37c973731fe4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "openshift-service-ca.crt", "reconcileID": "5e32860f-b949-4375-a2ef-37c973731fe4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b81136d3-2a89-4090-b834-4347b7a779fc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a6d4b797"}, "namespace": "ai-tenant-e2e-mt-a6d4b797", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b81136d3-2a89-4090-b834-4347b7a779fc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a6d4b797"} 2026-07-07T19:51:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "openshift-service-ca.crt", "reconcileID": "47d5196b-ac6a-4e80-bded-32f4017e5a6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-trusted-ca-bundle", "reconcileID": "31231660-da89-4351-bcf8-72dff9f14267", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "openshift-service-ca.crt", "reconcileID": "151af744-0e7a-4042-aaa0-608e902637d1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-5c180a55", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "openshift-service-ca.crt", "reconcileID": "151af744-0e7a-4042-aaa0-608e902637d1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fd0b52b0-2dc9-4770-86fe-827035b59879", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "openshift-service-ca.crt", "reconcileID": "1d4bc91c-eeed-4a48-84bb-52cdb06f8896", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-trusted-ca-bundle", "reconcileID": "429c61f2-62ad-4485-951f-33fa9f38ba6c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-trusted-ca-bundle", "reconcileID": "429c61f2-62ad-4485-951f-33fa9f38ba6c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1ba6e44-5784-4722-8002-2e947c25ba80", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8637221e-87eb-4765-872f-4345beb4695d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8637221e-87eb-4765-872f-4345beb4695d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:11Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8637221e-87eb-4765-872f-4345beb4695d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-5c180a55 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:51:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-trusted-ca-bundle", "reconcileID": "1131315f-a2d1-4721-948d-aab68ea8a142", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-trusted-ca-bundle", "reconcileID": "1131315f-a2d1-4721-948d-aab68ea8a142", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "openshift-service-ca.crt", "reconcileID": "2b718538-6ddb-4b38-add8-6ea05263996a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "openshift-service-ca.crt", "reconcileID": "2b718538-6ddb-4b38-add8-6ea05263996a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "16ee4487-ff28-4622-9ce1-c528f899a9c3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-5c180a55"}, "namespace": "e2e-webhook-sub-5c180a55", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "16ee4487-ff28-4622-9ce1-c528f899a9c3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-5c180a55"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "openshift-service-ca.crt", "reconcileID": "3e2dfbb8-e301-4136-a67b-303c75d1919f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-trusted-ca-bundle", "reconcileID": "98602f90-26ec-4561-be91-31e251073cfd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "openshift-service-ca.crt", "reconcileID": "a64120a2-447d-41b0-9c3b-0ce7033b0253", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-6acc7b25", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "openshift-service-ca.crt", "reconcileID": "a64120a2-447d-41b0-9c3b-0ce7033b0253", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dd3bb5e8-55cf-4d4f-94f0-2904a6ebee75", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "openshift-service-ca.crt", "reconcileID": "450da356-d4ca-417d-80df-d9c0db36449c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-trusted-ca-bundle", "reconcileID": "0561623d-9c17-45e4-9b54-69a87ee45842", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-trusted-ca-bundle", "reconcileID": "0561623d-9c17-45e4-9b54-69a87ee45842", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "16c3b405-71d4-4b0b-80ee-8ee39a01983e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "62486d30-7120-41ac-a0a8-7de1bbc099a8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "62486d30-7120-41ac-a0a8-7de1bbc099a8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:17Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "62486d30-7120-41ac-a0a8-7de1bbc099a8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-6acc7b25 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:51:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-trusted-ca-bundle", "reconcileID": "35900672-931a-4720-bc0d-675641191f41", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-trusted-ca-bundle", "reconcileID": "35900672-931a-4720-bc0d-675641191f41", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "openshift-service-ca.crt", "reconcileID": "38af7076-cf87-405b-bbb2-b7732195694f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "openshift-service-ca.crt", "reconcileID": "38af7076-cf87-405b-bbb2-b7732195694f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1ff8db66-62a0-4145-922a-62bc7c8a0528", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-6acc7b25"}, "namespace": "e2e-webhook-auth-6acc7b25", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1ff8db66-62a0-4145-922a-62bc7c8a0528", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-6acc7b25"} 2026-07-07T19:51:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "4653aaf5-d4c1-4ca7-8304-014516f7fe8e", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "4653aaf5-d4c1-4ca7-8304-014516f7fe8e", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "4653aaf5-d4c1-4ca7-8304-014516f7fe8e", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "1983a0c6-75e0-46bb-bdb8-74090af96ca0", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "1983a0c6-75e0-46bb-bdb8-74090af96ca0", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "1983a0c6-75e0-46bb-bdb8-74090af96ca0", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "17ea0f8a-9f32-429b-9845-3eef0fcf5c69", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "17ea0f8a-9f32-429b-9845-3eef0fcf5c69", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "17ea0f8a-9f32-429b-9845-3eef0fcf5c69", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "39a5ed52-c0a7-4cb2-9a3a-6483060d9d9b", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "39a5ed52-c0a7-4cb2-9a3a-6483060d9d9b", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "39a5ed52-c0a7-4cb2-9a3a-6483060d9d9b", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "07b23010-afaa-485e-9b41-c568682e70f4", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "07b23010-afaa-485e-9b41-c568682e70f4", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "07b23010-afaa-485e-9b41-c568682e70f4", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "23949f33-7d68-4a65-aaeb-1a7428a59a5b", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "23949f33-7d68-4a65-aaeb-1a7428a59a5b", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "23949f33-7d68-4a65-aaeb-1a7428a59a5b", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "openshift-service-ca.crt", "reconcileID": "8101ceec-596c-401e-939c-5a25851d4119", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-trusted-ca-bundle", "reconcileID": "7b37fbae-827d-401d-a6a0-7f679d63b196", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "openshift-service-ca.crt", "reconcileID": "390bc7f0-abd9-49ab-a407-4d3e8ff54b38", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c9eca85c", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "openshift-service-ca.crt", "reconcileID": "390bc7f0-abd9-49ab-a407-4d3e8ff54b38", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "109564f6-864a-4db4-a8c9-b1d1ba8a1456", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "openshift-service-ca.crt", "reconcileID": "9f0ffe46-cc54-4964-b52d-9b71b6f4f053", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-trusted-ca-bundle", "reconcileID": "b3ccb83b-96d3-4d93-81d3-1d42c677e57d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:51:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-trusted-ca-bundle", "reconcileID": "b3ccb83b-96d3-4d93-81d3-1d42c677e57d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "69269977-3315-4cb6-ab13-8655663c4177", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:51:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "fcd617d4-0939-470a-9a54-b85d72c4add1", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "fcd617d4-0939-470a-9a54-b85d72c4add1", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "fcd617d4-0939-470a-9a54-b85d72c4add1", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "cd0f2331-8d48-4114-99ce-1726a951c4a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:51:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "8f56a102-e7ac-45fc-88e7-293ce2f02563", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:51:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "8f56a102-e7ac-45fc-88e7-293ce2f02563", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:51:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "8f56a102-e7ac-45fc-88e7-293ce2f02563", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:53:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "6cafea65-5a6e-491a-bc9e-239daf03de74", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:54:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "4b1c351d-4794-4eb4-8588-a43d9253324d", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:54:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "b78ce7cc-6518-4a05-9fcd-50a5ba588b9f", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:54:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "b78ce7cc-6518-4a05-9fcd-50a5ba588b9f", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c-authn-ssl"} 2026-07-07T19:54:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-c9eca85c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-c9eca85c", "reconcileID": "b78ce7cc-6518-4a05-9fcd-50a5ba588b9f", "Gateway": "e2e-mt-c9eca85c", "namespace": "openshift-ingress"} 2026-07-07T19:54:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6c0597e6-1c82-441b-a7c2-cb808ce92002", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:43Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6c0597e6-1c82-441b-a7c2-cb808ce92002", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:54:43Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6c0597e6-1c82-441b-a7c2-cb808ce92002", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c9eca85c because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:54:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-trusted-ca-bundle", "reconcileID": "6f6c3dfe-bc5c-4ced-8fdb-b974c3c8c650", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-trusted-ca-bundle", "reconcileID": "6f6c3dfe-bc5c-4ced-8fdb-b974c3c8c650", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "openshift-service-ca.crt", "reconcileID": "1c85421c-9752-4463-a8a6-5dad11b09813", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "openshift-service-ca.crt", "reconcileID": "1c85421c-9752-4463-a8a6-5dad11b09813", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a4e91672-f6cf-4b21-a5d2-ddd8a34857c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c9eca85c"}, "namespace": "ai-tenant-e2e-mt-c9eca85c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a4e91672-f6cf-4b21-a5d2-ddd8a34857c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c9eca85c"} 2026-07-07T19:54:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "openshift-service-ca.crt", "reconcileID": "7c598964-6034-4c12-82a5-adfb7e9b23f5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-trusted-ca-bundle", "reconcileID": "212104da-7f55-4324-a92e-3ca46b7fa729", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "openshift-service-ca.crt", "reconcileID": "dcb181e7-e99d-482a-a759-647845c4a127", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1702a8cf", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:54:46Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "openshift-service-ca.crt", "reconcileID": "dcb181e7-e99d-482a-a759-647845c4a127", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c9d1135-171e-423b-b01e-d7cb51722ff4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "openshift-service-ca.crt", "reconcileID": "6789d252-5f29-4d65-8ca3-9134ff56d2e4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-trusted-ca-bundle", "reconcileID": "94d85232-a527-4870-9542-9d4f69ce8666", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:54:46Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-trusted-ca-bundle", "reconcileID": "94d85232-a527-4870-9542-9d4f69ce8666", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9bc09e8d-fc0f-4d62-8020-2d26a68257bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "openshift-service-ca.crt", "reconcileID": "a4e89e67-f5e9-4cf6-9817-9dfdd03876bc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-trusted-ca-bundle", "reconcileID": "3051501c-3576-406b-8038-42ab7912da93", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:47Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "openshift-service-ca.crt", "reconcileID": "3ca98fdc-9889-45ef-b6d0-7eb82113f245", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4f503759", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:54:47Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "openshift-service-ca.crt", "reconcileID": "3ca98fdc-9889-45ef-b6d0-7eb82113f245", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fc596dc2-c460-4830-a08c-fc334a0d4d22", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "openshift-service-ca.crt", "reconcileID": "66f234c5-4b74-49e5-9b1f-afdebd878e25", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:47Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-trusted-ca-bundle", "reconcileID": "a5f61e95-cd9f-44f2-a275-521094f57d13", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:54:47Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-trusted-ca-bundle", "reconcileID": "a5f61e95-cd9f-44f2-a275-521094f57d13", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3eb35d87-29a7-4df4-90ed-4c761c2a038a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:54:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "961732bf-8c8c-4f0d-8c95-60b160f23581", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "961732bf-8c8c-4f0d-8c95-60b160f23581", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:54:55Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "961732bf-8c8c-4f0d-8c95-60b160f23581", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-1702a8cf because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:54:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-trusted-ca-bundle", "reconcileID": "7bdbb18c-f01c-420b-ac7c-c017415c2c1c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-trusted-ca-bundle", "reconcileID": "7bdbb18c-f01c-420b-ac7c-c017415c2c1c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "openshift-service-ca.crt", "reconcileID": "e7725d64-1a6a-4145-a7fc-f0ccfa8c88e2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "openshift-service-ca.crt", "reconcileID": "e7725d64-1a6a-4145-a7fc-f0ccfa8c88e2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8bb77e57-db19-4506-921f-02973d7ad480", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:54:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1702a8cf"}, "namespace": "ai-tenant-e2e-mt-1702a8cf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8bb77e57-db19-4506-921f-02973d7ad480", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1702a8cf"} 2026-07-07T19:55:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a22952b3-9202-45b4-9621-6458af7c4802", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a22952b3-9202-45b4-9621-6458af7c4802", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:02Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a22952b3-9202-45b4-9621-6458af7c4802", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-4f503759 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:55:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-trusted-ca-bundle", "reconcileID": "885371e6-ee46-4fce-a085-c5b756024b9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-trusted-ca-bundle", "reconcileID": "885371e6-ee46-4fce-a085-c5b756024b9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "openshift-service-ca.crt", "reconcileID": "7d089d71-e9c4-4fb9-a54e-fa2fb1527cf6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "openshift-service-ca.crt", "reconcileID": "7d089d71-e9c4-4fb9-a54e-fa2fb1527cf6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c798737e-f36a-4be3-8dad-6b7f5eac1d4b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4f503759"}, "namespace": "ai-tenant-e2e-mt-4f503759", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c798737e-f36a-4be3-8dad-6b7f5eac1d4b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4f503759"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "openshift-service-ca.crt", "reconcileID": "a3c0014b-65b1-4d65-9b7c-0729931f9bbe", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-trusted-ca-bundle", "reconcileID": "b205de53-9c58-42ef-a607-0a0c63972b74", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "openshift-service-ca.crt", "reconcileID": "27c4edf5-d907-41a2-a49c-97eb006f83b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c207005b", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "openshift-service-ca.crt", "reconcileID": "27c4edf5-d907-41a2-a49c-97eb006f83b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1a9ddad-5b4e-4207-a33a-79985c829145", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "openshift-service-ca.crt", "reconcileID": "6aee83e5-6da7-4b86-afb0-d2d050226b22", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-trusted-ca-bundle", "reconcileID": "19b6e9cd-197e-46aa-a3bc-2a831b7b57bb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-trusted-ca-bundle", "reconcileID": "19b6e9cd-197e-46aa-a3bc-2a831b7b57bb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2d3f159c-2e36-4045-b18b-d6889aa6a4d5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f666727e-9ae2-4d46-9c3f-4d8514dbcc09", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f666727e-9ae2-4d46-9c3f-4d8514dbcc09", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:41Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f666727e-9ae2-4d46-9c3f-4d8514dbcc09", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c207005b because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:55:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-trusted-ca-bundle", "reconcileID": "bfa28714-b6b2-4086-9894-f0d05baf7ee4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-trusted-ca-bundle", "reconcileID": "bfa28714-b6b2-4086-9894-f0d05baf7ee4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "openshift-service-ca.crt", "reconcileID": "6c115a84-b931-4f92-bdfa-f82235bd37fa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "openshift-service-ca.crt", "reconcileID": "6c115a84-b931-4f92-bdfa-f82235bd37fa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0610502e-9045-4d94-bd04-93b23b63587a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c207005b"}, "namespace": "ai-tenant-e2e-mt-c207005b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0610502e-9045-4d94-bd04-93b23b63587a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c207005b"} 2026-07-07T19:55:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "8f3fae3d-2633-4f72-999f-ac135287e078", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "8f3fae3d-2633-4f72-999f-ac135287e078", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "8f3fae3d-2633-4f72-999f-ac135287e078", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "604e265e-9544-4c95-84c7-604d775c417f", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "604e265e-9544-4c95-84c7-604d775c417f", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "604e265e-9544-4c95-84c7-604d775c417f", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "d516c35c-929e-4fc4-80e7-7cff11c62dfc", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "d516c35c-929e-4fc4-80e7-7cff11c62dfc", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "d516c35c-929e-4fc4-80e7-7cff11c62dfc", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "374bb089-84d4-418e-a311-a94155583d79", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "374bb089-84d4-418e-a311-a94155583d79", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "374bb089-84d4-418e-a311-a94155583d79", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "5becf486-dce8-48b9-b849-1a1a7e9456ab", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "5becf486-dce8-48b9-b849-1a1a7e9456ab", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "5becf486-dce8-48b9-b849-1a1a7e9456ab", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "7cae61df-9401-4021-b474-0c1eca1c8de6", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "7cae61df-9401-4021-b474-0c1eca1c8de6", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "7cae61df-9401-4021-b474-0c1eca1c8de6", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "a08cebb4-f431-4603-85b6-116fa2d8adf4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-trusted-ca-bundle", "reconcileID": "6cafda57-f66e-4dc8-8c8e-83677a324292", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "78bc21be-6202-4a8c-82fc-2200fc7fb70c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:45Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "78bc21be-6202-4a8c-82fc-2200fc7fb70c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "07eef589-1f06-4c4b-ade7-866a7531162d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:45Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "07eef589-1f06-4c4b-ade7-866a7531162d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:55:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d140b423-b7fe-496c-8bc0-01af029263de", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "c2ec4082-63e5-4e4c-8570-1318044869d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-trusted-ca-bundle", "reconcileID": "1a68974b-1747-46aa-b633-6e09842d4ae6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51", "update": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:55:45Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-trusted-ca-bundle", "reconcileID": "1a68974b-1747-46aa-b633-6e09842d4ae6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "442dc297-cbd8-4c14-b8d5-1e55d85cea8b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:55:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "cac81d0c-17bb-4769-9afa-d1bcee6bd747", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "cac81d0c-17bb-4769-9afa-d1bcee6bd747", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "cac81d0c-17bb-4769-9afa-d1bcee6bd747", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "e0777527-d3a5-4489-ae54-b0678ec1e9b8", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:55:53Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "552bc957-4b77-49b5-9f7a-6b1951d53435", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:55:53Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "552bc957-4b77-49b5-9f7a-6b1951d53435", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:55:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "552bc957-4b77-49b5-9f7a-6b1951d53435", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:58:38Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:58:38Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:58:40Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:58:40Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-07T19:58:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "a6f0334d-f3d7-42e2-81fa-12d1191852e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:58:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"opendatahub"}, "namespace": "opendatahub", "name": "openshift-service-ca.crt", "reconcileID": "b00cf193-dca4-42e2-a429-c01cb58185a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "opendatahub"} 2026-07-07T19:58:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "2ee74334-cf26-4709-934b-76a64965a524", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:58:56Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "2ee74334-cf26-4709-934b-76a64965a524", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51-authn-ssl"} 2026-07-07T19:58:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-00fc51","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-00fc51", "reconcileID": "2ee74334-cf26-4709-934b-76a64965a524", "Gateway": "e2e-inf-a-00fc51", "namespace": "openshift-ingress"} 2026-07-07T19:59:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "84e4ed6a-dfdd-4be1-99af-06ad3715488e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:59:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "84e4ed6a-dfdd-4be1-99af-06ad3715488e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51", "create": "odh-kserve-custom-ca-bundle"} 2026-07-07T19:59:00Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "84e4ed6a-dfdd-4be1-99af-06ad3715488e", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-00fc51 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-07T19:59:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-trusted-ca-bundle", "reconcileID": "2ac60031-f622-42a3-a609-1a952aea7575", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:59:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-trusted-ca-bundle", "reconcileID": "2ac60031-f622-42a3-a609-1a952aea7575", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:59:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "033af7a8-342f-4e90-a852-8618a852c1c8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:59:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "openshift-service-ca.crt", "reconcileID": "033af7a8-342f-4e90-a852-8618a852c1c8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:59:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "39caef2c-d5d8-4c4d-acd6-377c024b7739", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"} 2026-07-07T19:59:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-00fc51"}, "namespace": "ai-tenant-e2e-inf-a-00fc51", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "39caef2c-d5d8-4c4d-acd6-377c024b7739", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-00fc51"}